From 0cb09ce62f4a3ebc2cc8fe74af86e011c58a9f25 Mon Sep 17 00:00:00 2001
From: Chuyang Deng <chuyangd@amazon.com>
Date: Mon, 27 Apr 2020 18:34:43 -0700
Subject: [PATCH 01/22] infra: use tox in buildspec

---
 buildspec.yml | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/buildspec.yml b/buildspec.yml
index d9e70ebd..42e9ba2c 100644
--- a/buildspec.yml
+++ b/buildspec.yml
@@ -16,7 +16,7 @@ env:
     ECR_REPO: 'sagemaker-test'
     GITHUB_REPO: 'sagemaker-pytorch-serving-container'
     SETUP_FILE: 'setup_cmds.sh'
-    SETUP_CMDS: '#!/bin/bash\npip install --upgrade pip\npip install -U -e .\npip install -U -e .[test]'
+    SETUP_CMDS: '#!/bin/bash\npip3.6 install --upgrade pip\npip install -U -e .\npip3.6 install -U -e .[test]'
 
 
 phases:
@@ -85,8 +85,8 @@ phases:
       - launch-ec2-instance --instance-type $instance_type --ami-name dlami-ubuntu
 
       # run cpu integration tests
-      - py3_cmd="pytest test/integration/local --region $AWS_DEFAULT_REGION --docker-base-name $PREPROD_IMAGE --framework-version $FRAMEWORK_VERSION --py-version $CPU_PY3_VERSION --processor cpu --tag $CPU_PY3_TAG"
-      - py2_cmd="pytest test/integration/local --region $AWS_DEFAULT_REGION --docker-base-name $PREPROD_IMAGE --framework-version $FRAMEWORK_VERSION --py-version $CPU_PY2_VERSION --processor cpu --tag $CPU_PY2_TAG"
+      - py3_cmd="tox -e py36 test/integration/local --region $AWS_DEFAULT_REGION --docker-base-name $PREPROD_IMAGE --framework-version $FRAMEWORK_VERSION --py-version $CPU_PY3_VERSION --processor cpu --tag $CPU_PY3_TAG"
+      - py2_cmd="tox -e py36 test/integration/local --region $AWS_DEFAULT_REGION --docker-base-name $PREPROD_IMAGE --framework-version $FRAMEWORK_VERSION --py-version $CPU_PY2_VERSION --processor cpu --tag $CPU_PY2_TAG"
       - execute-command-if-has-matching-changes "$py3_cmd" "test/" "docker/*" "buildspec.yml"
       - execute-command-if-has-matching-changes "$py2_cmd" "test/" "docker/*" "buildspec.yml"
 
@@ -101,19 +101,19 @@ phases:
       - execute-command-if-has-matching-changes "$py2_cmd" "test/" "docker/*" "buildspec.yml"
 
       # run cpu sagemaker tests
-      - py3_cmd="pytest test/integration/sagemaker --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --aws-id $ACCOUNT --framework-version $FRAMEWORK_VERSION --py-version $CPU_PY3_VERSION --processor cpu --instance-type $CPU_INSTANCE_TYPE --tag $CPU_PY3_TAG"
-      - py2_cmd="pytest test/integration/sagemaker --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --aws-id $ACCOUNT --framework-version $FRAMEWORK_VERSION --py-version $CPU_PY2_VERSION --processor cpu --instance-type $CPU_INSTANCE_TYPE --tag $CPU_PY2_TAG"
+      - py3_cmd="tox -e py36 test/integration/sagemaker --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --aws-id $ACCOUNT --framework-version $FRAMEWORK_VERSION --py-version $CPU_PY3_VERSION --processor cpu --instance-type $CPU_INSTANCE_TYPE --tag $CPU_PY3_TAG"
+      - py2_cmd="tox -e py36 test/integration/sagemaker --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --aws-id $ACCOUNT --framework-version $FRAMEWORK_VERSION --py-version $CPU_PY2_VERSION --processor cpu --instance-type $CPU_INSTANCE_TYPE --tag $CPU_PY2_TAG"
       - execute-command-if-has-matching-changes "$py3_cmd" "test/" "docker/*" "buildspec.yml"
       - execute-command-if-has-matching-changes "$py2_cmd" "test/" "docker/*" "buildspec.yml"
 
       # run gpu sagemaker tests
-      - py3_cmd="pytest test/integration/sagemaker --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --aws-id $ACCOUNT --framework-version $FRAMEWORK_VERSION --py-version $GPU_PY3_VERSION --processor gpu --instance-type $GPU_INSTANCE_TYPE --tag $GPU_PY3_TAG"
-      - py2_cmd="pytest test/integration/sagemaker --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --aws-id $ACCOUNT --framework-version $FRAMEWORK_VERSION --py-version $GPU_PY2_VERSION --processor gpu --instance-type $GPU_INSTANCE_TYPE --tag $GPU_PY2_TAG"
+      - py3_cmd="tox -e py36 test/integration/sagemaker --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --aws-id $ACCOUNT --framework-version $FRAMEWORK_VERSION --py-version $GPU_PY3_VERSION --processor gpu --instance-type $GPU_INSTANCE_TYPE --tag $GPU_PY3_TAG"
+      - py2_cmd="tox -e py36 test/integration/sagemaker --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --aws-id $ACCOUNT --framework-version $FRAMEWORK_VERSION --py-version $GPU_PY2_VERSION --processor gpu --instance-type $GPU_INSTANCE_TYPE --tag $GPU_PY2_TAG"
       - execute-command-if-has-matching-changes "$py3_cmd" "test/" "docker/*" "buildspec.yml"
       - execute-command-if-has-matching-changes "$py2_cmd" "test/" "docker/*" "buildspec.yml"
 
       # run eia sagemaker tests
-      - py3_cmd="pytest test/integration/sagemaker --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --aws-id $ACCOUNT --framework-version $EIA_FRAMEWORK_VERSION --py-version $EIA_PY3_VERSION --processor cpu --instance-type $CPU_INSTANCE_TYPE --accelerator-type $EIA_ACCELERATOR_TYPE --tag $EIA_PY3_TAG"
+      - py3_cmd="tox -e py36 test/integration/sagemaker --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --aws-id $ACCOUNT --framework-version $EIA_FRAMEWORK_VERSION --py-version $EIA_PY3_VERSION --processor cpu --instance-type $CPU_INSTANCE_TYPE --accelerator-type $EIA_ACCELERATOR_TYPE --tag $EIA_PY3_TAG"
       - execute-command-if-has-matching-changes "$py3_cmd" "test/" "docker/*" "buildspec.yml"
 
     finally:

From ae0e80344c0736a3e972e64e14c91940c4b7c777 Mon Sep 17 00:00:00 2001
From: Chuyang Deng <chuyangd@amazon.com>
Date: Mon, 27 Apr 2020 20:27:51 -0700
Subject: [PATCH 02/22] add missing dashes

---
 buildspec.yml | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/buildspec.yml b/buildspec.yml
index 42e9ba2c..b9dd0712 100644
--- a/buildspec.yml
+++ b/buildspec.yml
@@ -85,8 +85,8 @@ phases:
       - launch-ec2-instance --instance-type $instance_type --ami-name dlami-ubuntu
 
       # run cpu integration tests
-      - py3_cmd="tox -e py36 test/integration/local --region $AWS_DEFAULT_REGION --docker-base-name $PREPROD_IMAGE --framework-version $FRAMEWORK_VERSION --py-version $CPU_PY3_VERSION --processor cpu --tag $CPU_PY3_TAG"
-      - py2_cmd="tox -e py36 test/integration/local --region $AWS_DEFAULT_REGION --docker-base-name $PREPROD_IMAGE --framework-version $FRAMEWORK_VERSION --py-version $CPU_PY2_VERSION --processor cpu --tag $CPU_PY2_TAG"
+      - py3_cmd="tox -e py36 -- test/integration/local --region $AWS_DEFAULT_REGION --docker-base-name $PREPROD_IMAGE --framework-version $FRAMEWORK_VERSION --py-version $CPU_PY3_VERSION --processor cpu --tag $CPU_PY3_TAG"
+      - py2_cmd="tox -e py36 -- test/integration/local --region $AWS_DEFAULT_REGION --docker-base-name $PREPROD_IMAGE --framework-version $FRAMEWORK_VERSION --py-version $CPU_PY2_VERSION --processor cpu --tag $CPU_PY2_TAG"
       - execute-command-if-has-matching-changes "$py3_cmd" "test/" "docker/*" "buildspec.yml"
       - execute-command-if-has-matching-changes "$py2_cmd" "test/" "docker/*" "buildspec.yml"
 
@@ -101,19 +101,19 @@ phases:
       - execute-command-if-has-matching-changes "$py2_cmd" "test/" "docker/*" "buildspec.yml"
 
       # run cpu sagemaker tests
-      - py3_cmd="tox -e py36 test/integration/sagemaker --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --aws-id $ACCOUNT --framework-version $FRAMEWORK_VERSION --py-version $CPU_PY3_VERSION --processor cpu --instance-type $CPU_INSTANCE_TYPE --tag $CPU_PY3_TAG"
-      - py2_cmd="tox -e py36 test/integration/sagemaker --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --aws-id $ACCOUNT --framework-version $FRAMEWORK_VERSION --py-version $CPU_PY2_VERSION --processor cpu --instance-type $CPU_INSTANCE_TYPE --tag $CPU_PY2_TAG"
+      - py3_cmd="tox -e py36 -- test/integration/sagemaker --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --aws-id $ACCOUNT --framework-version $FRAMEWORK_VERSION --py-version $CPU_PY3_VERSION --processor cpu --instance-type $CPU_INSTANCE_TYPE --tag $CPU_PY3_TAG"
+      - py2_cmd="tox -e py36 -- test/integration/sagemaker --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --aws-id $ACCOUNT --framework-version $FRAMEWORK_VERSION --py-version $CPU_PY2_VERSION --processor cpu --instance-type $CPU_INSTANCE_TYPE --tag $CPU_PY2_TAG"
       - execute-command-if-has-matching-changes "$py3_cmd" "test/" "docker/*" "buildspec.yml"
       - execute-command-if-has-matching-changes "$py2_cmd" "test/" "docker/*" "buildspec.yml"
 
       # run gpu sagemaker tests
-      - py3_cmd="tox -e py36 test/integration/sagemaker --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --aws-id $ACCOUNT --framework-version $FRAMEWORK_VERSION --py-version $GPU_PY3_VERSION --processor gpu --instance-type $GPU_INSTANCE_TYPE --tag $GPU_PY3_TAG"
-      - py2_cmd="tox -e py36 test/integration/sagemaker --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --aws-id $ACCOUNT --framework-version $FRAMEWORK_VERSION --py-version $GPU_PY2_VERSION --processor gpu --instance-type $GPU_INSTANCE_TYPE --tag $GPU_PY2_TAG"
+      - py3_cmd="tox -e py36 -- test/integration/sagemaker --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --aws-id $ACCOUNT --framework-version $FRAMEWORK_VERSION --py-version $GPU_PY3_VERSION --processor gpu --instance-type $GPU_INSTANCE_TYPE --tag $GPU_PY3_TAG"
+      - py2_cmd="tox -e py36 -- test/integration/sagemaker --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --aws-id $ACCOUNT --framework-version $FRAMEWORK_VERSION --py-version $GPU_PY2_VERSION --processor gpu --instance-type $GPU_INSTANCE_TYPE --tag $GPU_PY2_TAG"
       - execute-command-if-has-matching-changes "$py3_cmd" "test/" "docker/*" "buildspec.yml"
       - execute-command-if-has-matching-changes "$py2_cmd" "test/" "docker/*" "buildspec.yml"
 
       # run eia sagemaker tests
-      - py3_cmd="tox -e py36 test/integration/sagemaker --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --aws-id $ACCOUNT --framework-version $EIA_FRAMEWORK_VERSION --py-version $EIA_PY3_VERSION --processor cpu --instance-type $CPU_INSTANCE_TYPE --accelerator-type $EIA_ACCELERATOR_TYPE --tag $EIA_PY3_TAG"
+      - py3_cmd="tox -e py36 -- test/integration/sagemaker --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --aws-id $ACCOUNT --framework-version $EIA_FRAMEWORK_VERSION --py-version $EIA_PY3_VERSION --processor cpu --instance-type $CPU_INSTANCE_TYPE --accelerator-type $EIA_ACCELERATOR_TYPE --tag $EIA_PY3_TAG"
       - execute-command-if-has-matching-changes "$py3_cmd" "test/" "docker/*" "buildspec.yml"
 
     finally:

From 676a43f58974380caad248996bd8cb3a969a406c Mon Sep 17 00:00:00 2001
From: Chuyang Deng <chuyangd@amazon.com>
Date: Mon, 27 Apr 2020 21:30:34 -0700
Subject: [PATCH 03/22] install pyyaml as test dependency

---
 tox.ini | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tox.ini b/tox.ini
index fba5050a..5c94a4d1 100644
--- a/tox.ini
+++ b/tox.ini
@@ -67,6 +67,7 @@ deps =
     retrying
     six
     future
+    pyyaml
 
 [testenv:flake8]
 basepython = python3

From abca3d5346bdc7b4c450065f2fedd5a3b4dd4fa0 Mon Sep 17 00:00:00 2001
From: Chuyang Deng <chuyangd@amazon.com>
Date: Mon, 27 Apr 2020 22:35:58 -0700
Subject: [PATCH 04/22] update coverage file

---
 .coverage  | 20 ++++++++++++++++++++
 .gitignore |  1 -
 tox.ini    |  2 +-
 3 files changed, 21 insertions(+), 2 deletions(-)
 create mode 100644 .coverage

diff --git a/.coverage b/.coverage
new file mode 100644
index 00000000..61e1deb8
--- /dev/null
+++ b/.coverage
@@ -0,0 +1,20 @@
+[run]
+branch = True
+timid = True
+
+[report]
+exclude_lines =
+    pragma: no cover
+    pragma: py3 no cover
+    if six.PY2
+    elif six.PY2
+
+partial_branches =
+    pragma: no cover
+    pragma: py3 no cover
+    if six.PY3
+    elif six.PY3
+
+show_missing = True
+
+fail_under = 95
diff --git a/.gitignore b/.gitignore
index 4f89b30d..e7c6ae3c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,7 +1,6 @@
 dist/
 test/resources/local_mode_lock
 .tox/
-.coverage
 *.egg-info/
 __pycache__/
 .idea/
diff --git a/tox.ini b/tox.ini
index 5c94a4d1..4d6f3890 100644
--- a/tox.ini
+++ b/tox.ini
@@ -51,7 +51,7 @@ passenv =
     AWS_DEFAULT_REGION
 commands =
     coverage run --source sagemaker_pytorch_serving_container -m pytest {posargs}
-    {env:IGNORE_COVERAGE:} coverage report --fail-under=95 --include *sagemaker_pytorch_serving_container*
+    {env:IGNORE_COVERAGE:} coverage report --include *sagemaker_pytorch_serving_container* --show-missing
 deps =
     coverage
     pytest

From a2fa5cb88a483bac975c1b3548a5e5362d88e0a4 Mon Sep 17 00:00:00 2001
From: Chuyang Deng <chuyangd@amazon.com>
Date: Mon, 27 Apr 2020 23:46:10 -0700
Subject: [PATCH 05/22] update .coveragec file name

---
 .coverage => .coveragec | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename .coverage => .coveragec (100%)

diff --git a/.coverage b/.coveragec
similarity index 100%
rename from .coverage
rename to .coveragec

From 0668f2b6ef3b017ffea7749e5e1aca61267c5f33 Mon Sep 17 00:00:00 2001
From: Chuyang Deng <chuyangd@amazon.com>
Date: Tue, 28 Apr 2020 00:50:50 -0700
Subject: [PATCH 06/22] add coverage file

---
 tox.ini | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tox.ini b/tox.ini
index 4d6f3890..5a02a11a 100644
--- a/tox.ini
+++ b/tox.ini
@@ -50,7 +50,7 @@ passenv =
     AWS_CONTAINER_CREDENTIALS_RELATIVE_URI
     AWS_DEFAULT_REGION
 commands =
-    coverage run --source sagemaker_pytorch_serving_container -m pytest {posargs}
+    coverage run --rcfile .coveragerc_{envname} --source sagemaker_pytorch_serving_container -m pytest {posargs}
     {env:IGNORE_COVERAGE:} coverage report --include *sagemaker_pytorch_serving_container* --show-missing
 deps =
     coverage

From 59677e9d596075246f21c5623ecc4457bc909c5b Mon Sep 17 00:00:00 2001
From: Chuyang Deng <chuyangd@amazon.com>
Date: Tue, 28 Apr 2020 09:01:34 -0700
Subject: [PATCH 07/22] update coverage report name

---
 .coveragec => .coveragec_py36 | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename .coveragec => .coveragec_py36 (100%)

diff --git a/.coveragec b/.coveragec_py36
similarity index 100%
rename from .coveragec
rename to .coveragec_py36

From c83d882eec44bfb3f7dcbca3e6a719013ed80771 Mon Sep 17 00:00:00 2001
From: Chuyang Deng <chuyangd@amazon.com>
Date: Tue, 28 Apr 2020 09:23:39 -0700
Subject: [PATCH 08/22] update coverage command

---
 .coveragec_py36 => .coverage | 0
 tox.ini                      | 2 +-
 2 files changed, 1 insertion(+), 1 deletion(-)
 rename .coveragec_py36 => .coverage (100%)

diff --git a/.coveragec_py36 b/.coverage
similarity index 100%
rename from .coveragec_py36
rename to .coverage
diff --git a/tox.ini b/tox.ini
index 5a02a11a..4d6f3890 100644
--- a/tox.ini
+++ b/tox.ini
@@ -50,7 +50,7 @@ passenv =
     AWS_CONTAINER_CREDENTIALS_RELATIVE_URI
     AWS_DEFAULT_REGION
 commands =
-    coverage run --rcfile .coveragerc_{envname} --source sagemaker_pytorch_serving_container -m pytest {posargs}
+    coverage run --source sagemaker_pytorch_serving_container -m pytest {posargs}
     {env:IGNORE_COVERAGE:} coverage report --include *sagemaker_pytorch_serving_container* --show-missing
 deps =
     coverage

From fd926497d5400a28eade3db0ea78adcaad18a63a Mon Sep 17 00:00:00 2001
From: Chuyang Deng <chuyangd@amazon.com>
Date: Tue, 28 Apr 2020 09:31:00 -0700
Subject: [PATCH 09/22] update buildspec-toolkit

---
 .coverage => .coveragec_py36 |  0
 buildspec-toolkit.yml        | 22 +++++++++++-----------
 buildspec.yml                |  2 +-
 tox.ini                      |  5 +++--
 4 files changed, 15 insertions(+), 14 deletions(-)
 rename .coverage => .coveragec_py36 (100%)

diff --git a/.coverage b/.coveragec_py36
similarity index 100%
rename from .coverage
rename to .coveragec_py36
diff --git a/buildspec-toolkit.yml b/buildspec-toolkit.yml
index a1a2f8ed..f96ca709 100644
--- a/buildspec-toolkit.yml
+++ b/buildspec-toolkit.yml
@@ -11,7 +11,7 @@ env:
     GITHUB_REPO: 'sagemaker-pytorch-serving-container'
     DLC_ACCOUNT: '763104351884'
     SETUP_FILE: 'setup_cmds.sh'
-    SETUP_CMDS: '#!/bin/bash\npip install --upgrade pip\npip install -U -e .\npip install -U -e .[test]'
+    SETUP_CMDS: '#!/bin/bash\npip3.6 install --upgrade pip\npip3.6 install -U -e .\npip3.6 install -U -e .[test]'
 
 phases:
   pre_build:
@@ -29,7 +29,7 @@ phases:
       - PY_COLORS=0
 
       # install
-      - pip3 install -U -e .[test]
+      - pip3.6 install -U -e .[test]
 
       # run linters
       - tox -e flake8,twine
@@ -44,9 +44,9 @@ phases:
       - DLC_EIA_TAG="$FRAMEWORK_VERSION-dlc-eia-$BUILD_ID"
 
       # run local CPU integration tests (build and push the image to ECR repo)
-      - test_cmd="pytest test-toolkit/integration/local --build-image --push-image --dockerfile-type pytorch --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --aws-id $ACCOUNT --framework-version $FRAMEWORK_VERSION --processor cpu --tag $GENERIC_TAG"
+      - test_cmd="tox -e py36 -- test-toolkit/integration/local --build-image --push-image --dockerfile-type pytorch --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --aws-id $ACCOUNT --framework-version $FRAMEWORK_VERSION --processor cpu --tag $GENERIC_TAG"
       - execute-command-if-has-matching-changes "$test_cmd" "test-toolkit/" "src/*.py" "setup.py" "setup.cfg" "buildspec-toolkit.yml" "docker/build_artifacts/*"
-      - test_cmd="pytest test-toolkit/integration/local --build-image --push-image --dockerfile-type dlc.cpu --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --aws-id $ACCOUNT --framework-version $FRAMEWORK_VERSION --processor cpu --tag $DLC_CPU_TAG"
+      - test_cmd="tox -e py36 -- test-toolkit/integration/local --build-image --push-image --dockerfile-type dlc.cpu --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --aws-id $ACCOUNT --framework-version $FRAMEWORK_VERSION --processor cpu --tag $DLC_CPU_TAG"
       - execute-command-if-has-matching-changes "$test_cmd" "test-toolkit/" "src/*.py" "setup.py" "setup.cfg" "buildspec-toolkit.yml" "docker/build_artifacts/*"
 
       # launch remote GPU instance
@@ -67,27 +67,27 @@ phases:
       # run GPU local integration tests
       - printf "$SETUP_CMDS" > $SETUP_FILE
       # no reason to rebuild the image again since it was already built and pushed to ECR during CPU tests
-      - generic_cmd="pytest test-toolkit/integration/local --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --aws-id $ACCOUNT --framework-version $FRAMEWORK_VERSION --processor gpu --tag $GENERIC_TAG"
+      - generic_cmd="tox -e py36 -- test-toolkit/integration/local --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --aws-id $ACCOUNT --framework-version $FRAMEWORK_VERSION --processor gpu --tag $GENERIC_TAG"
       - test_cmd="remote-test --github-repo $GITHUB_REPO --test-cmd \"$generic_cmd\" --setup-file $SETUP_FILE --pr-number \"$PR_NUM\""
       - execute-command-if-has-matching-changes "$test_cmd" "test-toolkit/" "src/*.py" "setup.py" "setup.cfg" "buildspec-toolkit.yml" "docker/build_artifacts/*"
-      - dlc_cmd="pytest test-toolkit/integration/local --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --aws-id $ACCOUNT --framework-version $FRAMEWORK_VERSION --processor gpu --tag $DLC_GPU_TAG"
+      - dlc_cmd="tox -e py36 -- test-toolkit/integration/local --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --aws-id $ACCOUNT --framework-version $FRAMEWORK_VERSION --processor gpu --tag $DLC_GPU_TAG"
       - test_cmd="remote-test --github-repo $GITHUB_REPO --test-cmd \"$dlc_cmd\" --setup-file $SETUP_FILE --pr-number \"$PR_NUM\" --skip-setup"
       - execute-command-if-has-matching-changes "$test_cmd" "test-toolkit/" "src/*.py" "setup.py" "setup.cfg" "buildspec-toolkit.yml" "docker/build_artifacts/*"
 
       # run CPU sagemaker integration tests
-      - test_cmd="pytest test-toolkit/integration/sagemaker --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --aws-id $ACCOUNT --framework-version $FRAMEWORK_VERSION --processor cpu --instance-type $CPU_INSTANCE_TYPE --tag $GENERIC_TAG"
+      - test_cmd="tox -e py36 -- test-toolkit/integration/sagemaker --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --aws-id $ACCOUNT --framework-version $FRAMEWORK_VERSION --processor cpu --instance-type $CPU_INSTANCE_TYPE --tag $GENERIC_TAG"
       - execute-command-if-has-matching-changes "$test_cmd" "test-toolkit/" "src/*.py" "setup.py" "setup.cfg" "buildspec-toolkit.yml" "docker/build_artifacts/*"
-      - test_cmd="pytest test-toolkit/integration/sagemaker --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --aws-id $ACCOUNT --framework-version $FRAMEWORK_VERSION --processor cpu --instance-type $CPU_INSTANCE_TYPE --tag $DLC_CPU_TAG"
+      - test_cmd="tox -e py36 -- test-toolkit/integration/sagemaker --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --aws-id $ACCOUNT --framework-version $FRAMEWORK_VERSION --processor cpu --instance-type $CPU_INSTANCE_TYPE --tag $DLC_CPU_TAG"
       - execute-command-if-has-matching-changes "$test_cmd" "test-toolkit/" "src/*.py" "setup.py" "setup.cfg" "buildspec-toolkit.yml" "docker/build_artifacts/*"
 
       # run GPU sagemaker integration tests
-      - test_cmd="pytest test-toolkit/integration/sagemaker --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --aws-id $ACCOUNT --framework-version $FRAMEWORK_VERSION --processor gpu --instance-type $GPU_INSTANCE_TYPE --tag $GENERIC_TAG"
+      - test_cmd="tox -e py36 -- test-toolkit/integration/sagemaker --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --aws-id $ACCOUNT --framework-version $FRAMEWORK_VERSION --processor gpu --instance-type $GPU_INSTANCE_TYPE --tag $GENERIC_TAG"
       - execute-command-if-has-matching-changes "$test_cmd" "test-toolkit/" "src/*.py" "setup.py" "setup.cfg" "buildspec-toolkit.yml" "docker/build_artifacts/*"
-      - test_cmd="pytest test-toolkit/integration/sagemaker --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --aws-id $ACCOUNT --framework-version $FRAMEWORK_VERSION --processor gpu --instance-type $GPU_INSTANCE_TYPE --tag $DLC_GPU_TAG"
+      - test_cmd="tox -e py36 -- test-toolkit/integration/sagemaker --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --aws-id $ACCOUNT --framework-version $FRAMEWORK_VERSION --processor gpu --instance-type $GPU_INSTANCE_TYPE --tag $DLC_GPU_TAG"
       - execute-command-if-has-matching-changes "$test_cmd" "test-toolkit/" "src/*.py" "setup.py" "setup.cfg" "buildspec-toolkit.yml" "docker/build_artifacts/*"
 
       # run EIA sagemaker integration tests
-      - test_cmd="pytest test-toolkit/integration/sagemaker --build-image --push-image --dockerfile-type dlc.eia --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --aws-id $ACCOUNT --framework-version $EIA_FRAMEWORK_VERSION --processor cpu --instance-type $CPU_INSTANCE_TYPE --accelerator-type $EIA_ACCELERATOR_TYPE --tag $DLC_EIA_TAG"
+      - test_cmd="tox -e py36 -- test-toolkit/integration/sagemaker --build-image --push-image --dockerfile-type dlc.eia --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --aws-id $ACCOUNT --framework-version $EIA_FRAMEWORK_VERSION --processor cpu --instance-type $CPU_INSTANCE_TYPE --accelerator-type $EIA_ACCELERATOR_TYPE --tag $DLC_EIA_TAG"
       - execute-command-if-has-matching-changes "$test_cmd" "test-toolkit/" "src/*.py" "setup.py" "setup.cfg" "buildspec-toolkit.yml" "docker/build_artifacts/*"
 
     finally:
diff --git a/buildspec.yml b/buildspec.yml
index 2be99374..8ef9800b 100644
--- a/buildspec.yml
+++ b/buildspec.yml
@@ -74,7 +74,7 @@ phases:
 
       # run gpu integration tests
       - printf "$SETUP_CMDS" > $SETUP_FILE
-      - py3_pytest_cmd="pytest test/integration/local --region $AWS_DEFAULT_REGION --docker-base-name $PREPROD_IMAGE --framework-version $FRAMEWORK_VERSION --py-version $GPU_PY3_VERSION --processor gpu --tag $GPU_PY3_TAG"
+      - py3_pytest_cmd="tox -e py36 test/integration/local --region $AWS_DEFAULT_REGION --docker-base-name $PREPROD_IMAGE --framework-version $FRAMEWORK_VERSION --py-version $GPU_PY3_VERSION --processor gpu --tag $GPU_PY3_TAG"
       - py3_cmd="remote-test --github-repo $GITHUB_REPO --test-cmd \"$py3_pytest_cmd\" --setup-file $SETUP_FILE --pr-number \"$PR_NUM\""
       - execute-command-if-has-matching-changes "$py3_cmd" "test/" "docker/*" "buildspec.yml"
 
diff --git a/tox.ini b/tox.ini
index 4d6f3890..c514b48a 100644
--- a/tox.ini
+++ b/tox.ini
@@ -50,8 +50,9 @@ passenv =
     AWS_CONTAINER_CREDENTIALS_RELATIVE_URI
     AWS_DEFAULT_REGION
 commands =
-    coverage run --source sagemaker_pytorch_serving_container -m pytest {posargs}
-    {env:IGNORE_COVERAGE:} coverage report --include *sagemaker_pytorch_serving_container* --show-missing
+    coverage run --rcfile .coveragerc_{envname} --source sagemaker_pytorch_serving_container -m pytest {posargs}
+    {env:IGNORE_COVERAGE:} coverage report --rcfile .coveragerc_{envname}
+
 deps =
     coverage
     pytest

From e950e1ac7216b6d014822ee9c255cd12b0848ba2 Mon Sep 17 00:00:00 2001
From: Chuyang Deng <chuyangd@amazon.com>
Date: Tue, 28 Apr 2020 09:43:57 -0700
Subject: [PATCH 10/22] update coverage config file name

---
 .coveragec_py36 => .coveragerc_py36 | 0
 tox.ini                             | 2 +-
 2 files changed, 1 insertion(+), 1 deletion(-)
 rename .coveragec_py36 => .coveragerc_py36 (100%)

diff --git a/.coveragec_py36 b/.coveragerc_py36
similarity index 100%
rename from .coveragec_py36
rename to .coveragerc_py36
diff --git a/tox.ini b/tox.ini
index c514b48a..6a9ba0df 100644
--- a/tox.ini
+++ b/tox.ini
@@ -50,7 +50,7 @@ passenv =
     AWS_CONTAINER_CREDENTIALS_RELATIVE_URI
     AWS_DEFAULT_REGION
 commands =
-    coverage run --rcfile .coveragerc_{envname} --source sagemaker_pytorch_serving_container -m pytest {posargs}
+    coverage run --rcfile .coveragec_{envname} --source sagemaker_pytorch_serving_container -m pytest {posargs}
     {env:IGNORE_COVERAGE:} coverage report --rcfile .coveragerc_{envname}
 
 deps =

From 26bf7363a5c77fd7592b4ef9065d0c3ce3029e46 Mon Sep 17 00:00:00 2001
From: Chuyang Deng <chuyangd@amazon.com>
Date: Tue, 28 Apr 2020 09:52:22 -0700
Subject: [PATCH 11/22] update coverage command

---
 tox.ini | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tox.ini b/tox.ini
index 6a9ba0df..c514b48a 100644
--- a/tox.ini
+++ b/tox.ini
@@ -50,7 +50,7 @@ passenv =
     AWS_CONTAINER_CREDENTIALS_RELATIVE_URI
     AWS_DEFAULT_REGION
 commands =
-    coverage run --rcfile .coveragec_{envname} --source sagemaker_pytorch_serving_container -m pytest {posargs}
+    coverage run --rcfile .coveragerc_{envname} --source sagemaker_pytorch_serving_container -m pytest {posargs}
     {env:IGNORE_COVERAGE:} coverage report --rcfile .coveragerc_{envname}
 
 deps =

From 196b4356a6dbf4e46944307b3a1f259ccbd30147 Mon Sep 17 00:00:00 2001
From: Chuyang Deng <chuyangd@amazon.com>
Date: Tue, 28 Apr 2020 10:31:39 -0700
Subject: [PATCH 12/22] update tox command

---
 .coveragerc_py36 => .coveragerc | 0
 tox.ini                         | 4 ++--
 2 files changed, 2 insertions(+), 2 deletions(-)
 rename .coveragerc_py36 => .coveragerc (100%)

diff --git a/.coveragerc_py36 b/.coveragerc
similarity index 100%
rename from .coveragerc_py36
rename to .coveragerc
diff --git a/tox.ini b/tox.ini
index c514b48a..432fe6bd 100644
--- a/tox.ini
+++ b/tox.ini
@@ -50,8 +50,8 @@ passenv =
     AWS_CONTAINER_CREDENTIALS_RELATIVE_URI
     AWS_DEFAULT_REGION
 commands =
-    coverage run --rcfile .coveragerc_{envname} --source sagemaker_pytorch_serving_container -m pytest {posargs}
-    {env:IGNORE_COVERAGE:} coverage report --rcfile .coveragerc_{envname}
+    coverage run --source sagemaker_pytorch_serving_container -m pytest {posargs}
+    {env:IGNORE_COVERAGE:} coverage report --fail_under=90
 
 deps =
     coverage

From 9566704cbbe86ebb4d6b6ebf62c47e26b7ab3518 Mon Sep 17 00:00:00 2001
From: Chuyang Deng <chuyangd@amazon.com>
Date: Tue, 28 Apr 2020 11:09:43 -0700
Subject: [PATCH 13/22] fix a typo in coverage command

---
 tox.ini | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tox.ini b/tox.ini
index 432fe6bd..efdf2978 100644
--- a/tox.ini
+++ b/tox.ini
@@ -51,7 +51,7 @@ passenv =
     AWS_DEFAULT_REGION
 commands =
     coverage run --source sagemaker_pytorch_serving_container -m pytest {posargs}
-    {env:IGNORE_COVERAGE:} coverage report --fail_under=90
+    {env:IGNORE_COVERAGE:} coverage report --fail-under=90
 
 deps =
     coverage

From 833eac39a9b6754b926c552738c1b46362f9bcd7 Mon Sep 17 00:00:00 2001
From: Chuyang Deng <chuyangd@amazon.com>
Date: Tue, 28 Apr 2020 13:08:51 -0700
Subject: [PATCH 14/22] ignore coverage of integ tests

---
 buildspec-toolkit.yml | 18 +++++++++---------
 buildspec.yml         |  8 ++++----
 2 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/buildspec-toolkit.yml b/buildspec-toolkit.yml
index f96ca709..688f3def 100644
--- a/buildspec-toolkit.yml
+++ b/buildspec-toolkit.yml
@@ -44,9 +44,9 @@ phases:
       - DLC_EIA_TAG="$FRAMEWORK_VERSION-dlc-eia-$BUILD_ID"
 
       # run local CPU integration tests (build and push the image to ECR repo)
-      - test_cmd="tox -e py36 -- test-toolkit/integration/local --build-image --push-image --dockerfile-type pytorch --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --aws-id $ACCOUNT --framework-version $FRAMEWORK_VERSION --processor cpu --tag $GENERIC_TAG"
+      - test_cmd="IGNORE_COVERAGE=- tox -e py36 -- test-toolkit/integration/local --build-image --push-image --dockerfile-type pytorch --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --aws-id $ACCOUNT --framework-version $FRAMEWORK_VERSION --processor cpu --tag $GENERIC_TAG"
       - execute-command-if-has-matching-changes "$test_cmd" "test-toolkit/" "src/*.py" "setup.py" "setup.cfg" "buildspec-toolkit.yml" "docker/build_artifacts/*"
-      - test_cmd="tox -e py36 -- test-toolkit/integration/local --build-image --push-image --dockerfile-type dlc.cpu --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --aws-id $ACCOUNT --framework-version $FRAMEWORK_VERSION --processor cpu --tag $DLC_CPU_TAG"
+      - test_cmd="IGNORE_COVERAGE=- tox -e py36 -- test-toolkit/integration/local --build-image --push-image --dockerfile-type dlc.cpu --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --aws-id $ACCOUNT --framework-version $FRAMEWORK_VERSION --processor cpu --tag $DLC_CPU_TAG"
       - execute-command-if-has-matching-changes "$test_cmd" "test-toolkit/" "src/*.py" "setup.py" "setup.cfg" "buildspec-toolkit.yml" "docker/build_artifacts/*"
 
       # launch remote GPU instance
@@ -67,27 +67,27 @@ phases:
       # run GPU local integration tests
       - printf "$SETUP_CMDS" > $SETUP_FILE
       # no reason to rebuild the image again since it was already built and pushed to ECR during CPU tests
-      - generic_cmd="tox -e py36 -- test-toolkit/integration/local --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --aws-id $ACCOUNT --framework-version $FRAMEWORK_VERSION --processor gpu --tag $GENERIC_TAG"
+      - generic_cmd="IGNORE_COVERAGE=- tox -e py36 -- test-toolkit/integration/local --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --aws-id $ACCOUNT --framework-version $FRAMEWORK_VERSION --processor gpu --tag $GENERIC_TAG"
       - test_cmd="remote-test --github-repo $GITHUB_REPO --test-cmd \"$generic_cmd\" --setup-file $SETUP_FILE --pr-number \"$PR_NUM\""
       - execute-command-if-has-matching-changes "$test_cmd" "test-toolkit/" "src/*.py" "setup.py" "setup.cfg" "buildspec-toolkit.yml" "docker/build_artifacts/*"
-      - dlc_cmd="tox -e py36 -- test-toolkit/integration/local --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --aws-id $ACCOUNT --framework-version $FRAMEWORK_VERSION --processor gpu --tag $DLC_GPU_TAG"
+      - dlc_cmd="IGNORE_COVERAGE=- tox -e py36 -- test-toolkit/integration/local --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --aws-id $ACCOUNT --framework-version $FRAMEWORK_VERSION --processor gpu --tag $DLC_GPU_TAG"
       - test_cmd="remote-test --github-repo $GITHUB_REPO --test-cmd \"$dlc_cmd\" --setup-file $SETUP_FILE --pr-number \"$PR_NUM\" --skip-setup"
       - execute-command-if-has-matching-changes "$test_cmd" "test-toolkit/" "src/*.py" "setup.py" "setup.cfg" "buildspec-toolkit.yml" "docker/build_artifacts/*"
 
       # run CPU sagemaker integration tests
-      - test_cmd="tox -e py36 -- test-toolkit/integration/sagemaker --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --aws-id $ACCOUNT --framework-version $FRAMEWORK_VERSION --processor cpu --instance-type $CPU_INSTANCE_TYPE --tag $GENERIC_TAG"
+      - test_cmd="IGNORE_COVERAGE=- tox -e py36 -- test-toolkit/integration/sagemaker --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --aws-id $ACCOUNT --framework-version $FRAMEWORK_VERSION --processor cpu --instance-type $CPU_INSTANCE_TYPE --tag $GENERIC_TAG"
       - execute-command-if-has-matching-changes "$test_cmd" "test-toolkit/" "src/*.py" "setup.py" "setup.cfg" "buildspec-toolkit.yml" "docker/build_artifacts/*"
-      - test_cmd="tox -e py36 -- test-toolkit/integration/sagemaker --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --aws-id $ACCOUNT --framework-version $FRAMEWORK_VERSION --processor cpu --instance-type $CPU_INSTANCE_TYPE --tag $DLC_CPU_TAG"
+      - test_cmd="IGNORE_COVERAGE=- tox -e py36 -- test-toolkit/integration/sagemaker --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --aws-id $ACCOUNT --framework-version $FRAMEWORK_VERSION --processor cpu --instance-type $CPU_INSTANCE_TYPE --tag $DLC_CPU_TAG"
       - execute-command-if-has-matching-changes "$test_cmd" "test-toolkit/" "src/*.py" "setup.py" "setup.cfg" "buildspec-toolkit.yml" "docker/build_artifacts/*"
 
       # run GPU sagemaker integration tests
-      - test_cmd="tox -e py36 -- test-toolkit/integration/sagemaker --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --aws-id $ACCOUNT --framework-version $FRAMEWORK_VERSION --processor gpu --instance-type $GPU_INSTANCE_TYPE --tag $GENERIC_TAG"
+      - test_cmd="IGNORE_COVERAGE=- tox -e py36 -- test-toolkit/integration/sagemaker --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --aws-id $ACCOUNT --framework-version $FRAMEWORK_VERSION --processor gpu --instance-type $GPU_INSTANCE_TYPE --tag $GENERIC_TAG"
       - execute-command-if-has-matching-changes "$test_cmd" "test-toolkit/" "src/*.py" "setup.py" "setup.cfg" "buildspec-toolkit.yml" "docker/build_artifacts/*"
-      - test_cmd="tox -e py36 -- test-toolkit/integration/sagemaker --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --aws-id $ACCOUNT --framework-version $FRAMEWORK_VERSION --processor gpu --instance-type $GPU_INSTANCE_TYPE --tag $DLC_GPU_TAG"
+      - test_cmd="IGNORE_COVERAGE=- tox -e py36 -- test-toolkit/integration/sagemaker --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --aws-id $ACCOUNT --framework-version $FRAMEWORK_VERSION --processor gpu --instance-type $GPU_INSTANCE_TYPE --tag $DLC_GPU_TAG"
       - execute-command-if-has-matching-changes "$test_cmd" "test-toolkit/" "src/*.py" "setup.py" "setup.cfg" "buildspec-toolkit.yml" "docker/build_artifacts/*"
 
       # run EIA sagemaker integration tests
-      - test_cmd="tox -e py36 -- test-toolkit/integration/sagemaker --build-image --push-image --dockerfile-type dlc.eia --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --aws-id $ACCOUNT --framework-version $EIA_FRAMEWORK_VERSION --processor cpu --instance-type $CPU_INSTANCE_TYPE --accelerator-type $EIA_ACCELERATOR_TYPE --tag $DLC_EIA_TAG"
+      - test_cmd="IGNORE_COVERAGE=- tox -e py36 -- test-toolkit/integration/sagemaker --build-image --push-image --dockerfile-type dlc.eia --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --aws-id $ACCOUNT --framework-version $EIA_FRAMEWORK_VERSION --processor cpu --instance-type $CPU_INSTANCE_TYPE --accelerator-type $EIA_ACCELERATOR_TYPE --tag $DLC_EIA_TAG"
       - execute-command-if-has-matching-changes "$test_cmd" "test-toolkit/" "src/*.py" "setup.py" "setup.cfg" "buildspec-toolkit.yml" "docker/build_artifacts/*"
 
     finally:
diff --git a/buildspec.yml b/buildspec.yml
index 8ef9800b..d978a0a2 100644
--- a/buildspec.yml
+++ b/buildspec.yml
@@ -74,20 +74,20 @@ phases:
 
       # run gpu integration tests
       - printf "$SETUP_CMDS" > $SETUP_FILE
-      - py3_pytest_cmd="tox -e py36 test/integration/local --region $AWS_DEFAULT_REGION --docker-base-name $PREPROD_IMAGE --framework-version $FRAMEWORK_VERSION --py-version $GPU_PY3_VERSION --processor gpu --tag $GPU_PY3_TAG"
+      - py3_pytest_cmd="IGNORE_COVERAGE=- tox -e py36 test/integration/local --region $AWS_DEFAULT_REGION --docker-base-name $PREPROD_IMAGE --framework-version $FRAMEWORK_VERSION --py-version $GPU_PY3_VERSION --processor gpu --tag $GPU_PY3_TAG"
       - py3_cmd="remote-test --github-repo $GITHUB_REPO --test-cmd \"$py3_pytest_cmd\" --setup-file $SETUP_FILE --pr-number \"$PR_NUM\""
       - execute-command-if-has-matching-changes "$py3_cmd" "test/" "docker/*" "buildspec.yml"
 
       # run cpu sagemaker tests
-      - py3_cmd="tox -e py36 -- test/integration/sagemaker --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --aws-id $ACCOUNT --framework-version $FRAMEWORK_VERSION --py-version $CPU_PY3_VERSION --processor cpu --instance-type $CPU_INSTANCE_TYPE --tag $CPU_PY3_TAG"
+      - py3_cmd="IGNORE_COVERAGE=- tox -e py36 -- test/integration/sagemaker --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --aws-id $ACCOUNT --framework-version $FRAMEWORK_VERSION --py-version $CPU_PY3_VERSION --processor cpu --instance-type $CPU_INSTANCE_TYPE --tag $CPU_PY3_TAG"
       - execute-command-if-has-matching-changes "$py3_cmd" "test/" "docker/*" "buildspec.yml"
 
       # run gpu sagemaker tests
-      - py3_cmd="tox -e py36 -- test/integration/sagemaker --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --aws-id $ACCOUNT --framework-version $FRAMEWORK_VERSION --py-version $GPU_PY3_VERSION --processor gpu --instance-type $GPU_INSTANCE_TYPE --tag $GPU_PY3_TAG"
+      - py3_cmd="IGNORE_COVERAGE=- tox -e py36 -- test/integration/sagemaker --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --aws-id $ACCOUNT --framework-version $FRAMEWORK_VERSION --py-version $GPU_PY3_VERSION --processor gpu --instance-type $GPU_INSTANCE_TYPE --tag $GPU_PY3_TAG"
       - execute-command-if-has-matching-changes "$py3_cmd" "test/" "docker/*" "buildspec.yml"
 
       # run eia sagemaker tests
-      - py3_cmd="tox -e py36 -- test/integration/sagemaker --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --aws-id $ACCOUNT --framework-version $EIA_FRAMEWORK_VERSION --py-version $EIA_PY3_VERSION --processor cpu --instance-type $CPU_INSTANCE_TYPE --accelerator-type $EIA_ACCELERATOR_TYPE --tag $EIA_PY3_TAG"
+      - py3_cmd="IGNORE_COVERAGE=- tox -e py36 -- test/integration/sagemaker --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --aws-id $ACCOUNT --framework-version $EIA_FRAMEWORK_VERSION --py-version $EIA_PY3_VERSION --processor cpu --instance-type $CPU_INSTANCE_TYPE --accelerator-type $EIA_ACCELERATOR_TYPE --tag $EIA_PY3_TAG"
       - execute-command-if-has-matching-changes "$py3_cmd" "test/" "docker/*" "buildspec.yml"
 
     finally:

From 107d01f7e1ff3189046e6049fe7cd6c8afa34753 Mon Sep 17 00:00:00 2001
From: Chuyang Deng <chuyangd@amazon.com>
Date: Tue, 28 Apr 2020 13:09:42 -0700
Subject: [PATCH 15/22] ignore coverage of integ tests

---
 buildspec.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/buildspec.yml b/buildspec.yml
index d978a0a2..3396e2b0 100644
--- a/buildspec.yml
+++ b/buildspec.yml
@@ -32,7 +32,7 @@ phases:
       - tox -e flake8
 
       # install
-      - pip3 install -U -e .[test]
+      - pip3.6 install -U -e .[test]
 
       - cpu_dockerfile="Dockerfile.cpu"
       - gpu_dockerfile="Dockerfile.gpu"

From bb893e0dc4b17f248ccfb9750ba58d4091c3a47d Mon Sep 17 00:00:00 2001
From: Chuyang Deng <chuyangd@amazon.com>
Date: Tue, 28 Apr 2020 13:45:54 -0700
Subject: [PATCH 16/22] remove pip install

---
 buildspec-toolkit.yml | 5 -----
 buildspec.yml         | 5 -----
 2 files changed, 10 deletions(-)

diff --git a/buildspec-toolkit.yml b/buildspec-toolkit.yml
index 688f3def..7eb2fade 100644
--- a/buildspec-toolkit.yml
+++ b/buildspec-toolkit.yml
@@ -10,8 +10,6 @@ env:
     ECR_REPO: 'sagemaker-test'
     GITHUB_REPO: 'sagemaker-pytorch-serving-container'
     DLC_ACCOUNT: '763104351884'
-    SETUP_FILE: 'setup_cmds.sh'
-    SETUP_CMDS: '#!/bin/bash\npip3.6 install --upgrade pip\npip3.6 install -U -e .\npip3.6 install -U -e .[test]'
 
 phases:
   pre_build:
@@ -28,9 +26,6 @@ phases:
       - TOX_PARALLEL_NO_SPINNER=1
       - PY_COLORS=0
 
-      # install
-      - pip3.6 install -U -e .[test]
-
       # run linters
       - tox -e flake8,twine
 
diff --git a/buildspec.yml b/buildspec.yml
index 3396e2b0..e013e0c9 100644
--- a/buildspec.yml
+++ b/buildspec.yml
@@ -13,8 +13,6 @@ env:
     LOCAL_BASE_REPO: 'pytorch-base'
     ECR_REPO: 'sagemaker-test'
     GITHUB_REPO: 'sagemaker-pytorch-serving-container'
-    SETUP_FILE: 'setup_cmds.sh'
-    SETUP_CMDS: '#!/bin/bash\npip3.6 install --upgrade pip\npip install -U -e .\npip3.6 install -U -e .[test]'
 
 
 phases:
@@ -31,9 +29,6 @@ phases:
     commands:
       - tox -e flake8
 
-      # install
-      - pip3.6 install -U -e .[test]
-
       - cpu_dockerfile="Dockerfile.cpu"
       - gpu_dockerfile="Dockerfile.gpu"
       - eia_dockerfile="Dockerfile.eia"

From 086dd2f42912105546e148bc4e8bb1fd7808dddf Mon Sep 17 00:00:00 2001
From: Chuyang Deng <chuyangd@amazon.com>
Date: Tue, 28 Apr 2020 13:50:37 -0700
Subject: [PATCH 17/22] ignore coverage

---
 buildspec.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/buildspec.yml b/buildspec.yml
index e013e0c9..7c41938f 100644
--- a/buildspec.yml
+++ b/buildspec.yml
@@ -64,7 +64,7 @@ phases:
       - launch-ec2-instance --instance-type $instance_type --ami-name dlami-ubuntu
 
       # run cpu integration tests
-      - py3_cmd="tox -e py36 -- test/integration/local --region $AWS_DEFAULT_REGION --docker-base-name $PREPROD_IMAGE --framework-version $FRAMEWORK_VERSION --py-version $CPU_PY3_VERSION --processor cpu --tag $CPU_PY3_TAG"
+      - py3_cmd="IGNORE_COVERAGE=- tox -e py36 -- test/integration/local --region $AWS_DEFAULT_REGION --docker-base-name $PREPROD_IMAGE --framework-version $FRAMEWORK_VERSION --py-version $CPU_PY3_VERSION --processor cpu --tag $CPU_PY3_TAG"
       - execute-command-if-has-matching-changes "$py3_cmd" "test/" "docker/*" "buildspec.yml"
 
       # run gpu integration tests

From 5d2d051a46b05ddb9b2609d96cacfb793a3e1af4 Mon Sep 17 00:00:00 2001
From: Chuyang Deng <chuyangd@amazon.com>
Date: Tue, 28 Apr 2020 14:22:21 -0700
Subject: [PATCH 18/22] remove setup file

---
 buildspec-toolkit.yml | 5 ++---
 buildspec.yml         | 3 +--
 2 files changed, 3 insertions(+), 5 deletions(-)

diff --git a/buildspec-toolkit.yml b/buildspec-toolkit.yml
index 7eb2fade..10b26c27 100644
--- a/buildspec-toolkit.yml
+++ b/buildspec-toolkit.yml
@@ -60,13 +60,12 @@ phases:
       - docker push $PREPROD_IMAGE:$DLC_GPU_TAG
 
       # run GPU local integration tests
-      - printf "$SETUP_CMDS" > $SETUP_FILE
       # no reason to rebuild the image again since it was already built and pushed to ECR during CPU tests
       - generic_cmd="IGNORE_COVERAGE=- tox -e py36 -- test-toolkit/integration/local --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --aws-id $ACCOUNT --framework-version $FRAMEWORK_VERSION --processor gpu --tag $GENERIC_TAG"
-      - test_cmd="remote-test --github-repo $GITHUB_REPO --test-cmd \"$generic_cmd\" --setup-file $SETUP_FILE --pr-number \"$PR_NUM\""
+      - test_cmd="remote-test --github-repo $GITHUB_REPO --test-cmd \"$generic_cmd\" --pr-number \"$PR_NUM\""
       - execute-command-if-has-matching-changes "$test_cmd" "test-toolkit/" "src/*.py" "setup.py" "setup.cfg" "buildspec-toolkit.yml" "docker/build_artifacts/*"
       - dlc_cmd="IGNORE_COVERAGE=- tox -e py36 -- test-toolkit/integration/local --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --aws-id $ACCOUNT --framework-version $FRAMEWORK_VERSION --processor gpu --tag $DLC_GPU_TAG"
-      - test_cmd="remote-test --github-repo $GITHUB_REPO --test-cmd \"$dlc_cmd\" --setup-file $SETUP_FILE --pr-number \"$PR_NUM\" --skip-setup"
+      - test_cmd="remote-test --github-repo $GITHUB_REPO --test-cmd \"$dlc_cmd\" --pr-number \"$PR_NUM\" --skip-setup"
       - execute-command-if-has-matching-changes "$test_cmd" "test-toolkit/" "src/*.py" "setup.py" "setup.cfg" "buildspec-toolkit.yml" "docker/build_artifacts/*"
 
       # run CPU sagemaker integration tests
diff --git a/buildspec.yml b/buildspec.yml
index 7c41938f..f8a547e3 100644
--- a/buildspec.yml
+++ b/buildspec.yml
@@ -68,9 +68,8 @@ phases:
       - execute-command-if-has-matching-changes "$py3_cmd" "test/" "docker/*" "buildspec.yml"
 
       # run gpu integration tests
-      - printf "$SETUP_CMDS" > $SETUP_FILE
       - py3_pytest_cmd="IGNORE_COVERAGE=- tox -e py36 test/integration/local --region $AWS_DEFAULT_REGION --docker-base-name $PREPROD_IMAGE --framework-version $FRAMEWORK_VERSION --py-version $GPU_PY3_VERSION --processor gpu --tag $GPU_PY3_TAG"
-      - py3_cmd="remote-test --github-repo $GITHUB_REPO --test-cmd \"$py3_pytest_cmd\" --setup-file $SETUP_FILE --pr-number \"$PR_NUM\""
+      - py3_cmd="remote-test --github-repo $GITHUB_REPO --test-cmd \"$py3_pytest_cmd\" --pr-number \"$PR_NUM\""
       - execute-command-if-has-matching-changes "$py3_cmd" "test/" "docker/*" "buildspec.yml"
 
       # run cpu sagemaker tests

From 124eeecc868bd2127e0f124fd4bb3e039518e2c8 Mon Sep 17 00:00:00 2001
From: Chuyang Deng <chuyangd@amazon.com>
Date: Tue, 28 Apr 2020 15:00:06 -0700
Subject: [PATCH 19/22] use python -m pip install

---
 buildspec-toolkit.yml | 8 ++++++--
 buildspec.yml         | 5 ++++-
 2 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/buildspec-toolkit.yml b/buildspec-toolkit.yml
index 10b26c27..0bcc7a12 100644
--- a/buildspec-toolkit.yml
+++ b/buildspec-toolkit.yml
@@ -10,6 +10,9 @@ env:
     ECR_REPO: 'sagemaker-test'
     GITHUB_REPO: 'sagemaker-pytorch-serving-container'
     DLC_ACCOUNT: '763104351884'
+    SETUP_FILE: 'setup_cmds.sh'
+    SETUP_CMDS: '#!/bin/bash\npython3.6 -m pip install --upgrade pip\npython3.6 -m pip install -U -e .\npython3.6 -m pip install -U -e .[test]'
+
 
 phases:
   pre_build:
@@ -60,12 +63,13 @@ phases:
       - docker push $PREPROD_IMAGE:$DLC_GPU_TAG
 
       # run GPU local integration tests
+      - printf "$SETUP_CMDS" > $SETUP_FILE
       # no reason to rebuild the image again since it was already built and pushed to ECR during CPU tests
       - generic_cmd="IGNORE_COVERAGE=- tox -e py36 -- test-toolkit/integration/local --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --aws-id $ACCOUNT --framework-version $FRAMEWORK_VERSION --processor gpu --tag $GENERIC_TAG"
-      - test_cmd="remote-test --github-repo $GITHUB_REPO --test-cmd \"$generic_cmd\" --pr-number \"$PR_NUM\""
+      - test_cmd="remote-test --github-repo $GITHUB_REPO --test-cmd \"$generic_cmd\" --setup-file $SETUP_FILE --pr-number \"$PR_NUM\""
       - execute-command-if-has-matching-changes "$test_cmd" "test-toolkit/" "src/*.py" "setup.py" "setup.cfg" "buildspec-toolkit.yml" "docker/build_artifacts/*"
       - dlc_cmd="IGNORE_COVERAGE=- tox -e py36 -- test-toolkit/integration/local --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --aws-id $ACCOUNT --framework-version $FRAMEWORK_VERSION --processor gpu --tag $DLC_GPU_TAG"
-      - test_cmd="remote-test --github-repo $GITHUB_REPO --test-cmd \"$dlc_cmd\" --pr-number \"$PR_NUM\" --skip-setup"
+      - test_cmd="remote-test --github-repo $GITHUB_REPO --test-cmd \"$dlc_cmd\" --setup-file $SETUP_FILE --pr-number \"$PR_NUM\" --skip-setup"
       - execute-command-if-has-matching-changes "$test_cmd" "test-toolkit/" "src/*.py" "setup.py" "setup.cfg" "buildspec-toolkit.yml" "docker/build_artifacts/*"
 
       # run CPU sagemaker integration tests
diff --git a/buildspec.yml b/buildspec.yml
index f8a547e3..bbdf8aed 100644
--- a/buildspec.yml
+++ b/buildspec.yml
@@ -13,6 +13,8 @@ env:
     LOCAL_BASE_REPO: 'pytorch-base'
     ECR_REPO: 'sagemaker-test'
     GITHUB_REPO: 'sagemaker-pytorch-serving-container'
+    SETUP_FILE: 'setup_cmds.sh'
+    SETUP_CMDS: '#!/bin/bash\npython3.6 -m pip install --upgrade pip\npython3.6 -m pip install -U -e .\npython3.6 -m pip install -U -e .[test]'
 
 
 phases:
@@ -68,8 +70,9 @@ phases:
       - execute-command-if-has-matching-changes "$py3_cmd" "test/" "docker/*" "buildspec.yml"
 
       # run gpu integration tests
+      - printf "$SETUP_CMDS" > $SETUP_FILE
       - py3_pytest_cmd="IGNORE_COVERAGE=- tox -e py36 test/integration/local --region $AWS_DEFAULT_REGION --docker-base-name $PREPROD_IMAGE --framework-version $FRAMEWORK_VERSION --py-version $GPU_PY3_VERSION --processor gpu --tag $GPU_PY3_TAG"
-      - py3_cmd="remote-test --github-repo $GITHUB_REPO --test-cmd \"$py3_pytest_cmd\" --pr-number \"$PR_NUM\""
+      - py3_cmd="remote-test --github-repo $GITHUB_REPO --test-cmd \"$py3_pytest_cmd\" --setup-file $SETUP_FILE --pr-number \"$PR_NUM\""
       - execute-command-if-has-matching-changes "$py3_cmd" "test/" "docker/*" "buildspec.yml"
 
       # run cpu sagemaker tests

From 0f9dbae3851a04946468c4d0331f17598f561836 Mon Sep 17 00:00:00 2001
From: Chuyang Deng <chuyangd@amazon.com>
Date: Tue, 28 Apr 2020 16:30:52 -0700
Subject: [PATCH 20/22] fix a typo in tox command

---
 buildspec.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/buildspec.yml b/buildspec.yml
index bbdf8aed..b99a0f58 100644
--- a/buildspec.yml
+++ b/buildspec.yml
@@ -71,7 +71,7 @@ phases:
 
       # run gpu integration tests
       - printf "$SETUP_CMDS" > $SETUP_FILE
-      - py3_pytest_cmd="IGNORE_COVERAGE=- tox -e py36 test/integration/local --region $AWS_DEFAULT_REGION --docker-base-name $PREPROD_IMAGE --framework-version $FRAMEWORK_VERSION --py-version $GPU_PY3_VERSION --processor gpu --tag $GPU_PY3_TAG"
+      - py3_pytest_cmd="IGNORE_COVERAGE=- tox -e py36 -- test/integration/local --region $AWS_DEFAULT_REGION --docker-base-name $PREPROD_IMAGE --framework-version $FRAMEWORK_VERSION --py-version $GPU_PY3_VERSION --processor gpu --tag $GPU_PY3_TAG"
       - py3_cmd="remote-test --github-repo $GITHUB_REPO --test-cmd \"$py3_pytest_cmd\" --setup-file $SETUP_FILE --pr-number \"$PR_NUM\""
       - execute-command-if-has-matching-changes "$py3_cmd" "test/" "docker/*" "buildspec.yml"
 

From 9437c8affabea2e487114a39a61d5c999de08c24 Mon Sep 17 00:00:00 2001
From: Chuyang Deng <chuyangd@amazon.com>
Date: Tue, 28 Apr 2020 18:12:30 -0700
Subject: [PATCH 21/22] use coverage config file

---
 tox.ini | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tox.ini b/tox.ini
index efdf2978..937ff546 100644
--- a/tox.ini
+++ b/tox.ini
@@ -50,8 +50,8 @@ passenv =
     AWS_CONTAINER_CREDENTIALS_RELATIVE_URI
     AWS_DEFAULT_REGION
 commands =
-    coverage run --source sagemaker_pytorch_serving_container -m pytest {posargs}
-    {env:IGNORE_COVERAGE:} coverage report --fail-under=90
+    coverage run --rcfile .coveragerc --source sagemaker_pytorch_serving_container -m pytest {posargs}
+    {env:IGNORE_COVERAGE:} coverage report --fail-under=90 --include *sagemaker_pytorch_serving_container*
 
 deps =
     coverage

From 8d7a5bc026b95c7abb3c5aae4d24daa7b9c81ac4 Mon Sep 17 00:00:00 2001
From: Chuyang Deng <chuyangd@amazon.com>
Date: Wed, 29 Apr 2020 09:52:12 -0700
Subject: [PATCH 22/22] update coverage fail shreshold to 90

---
 .coveragerc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.coveragerc b/.coveragerc
index 61e1deb8..96bb72bf 100644
--- a/.coveragerc
+++ b/.coveragerc
@@ -17,4 +17,4 @@ partial_branches =
 
 show_missing = True
 
-fail_under = 95
+fail_under = 90