diff --git a/buildspec-slowtests.yml b/buildspec-slowtests.yml
index cfa9a66ede..0767db4a62 100644
--- a/buildspec-slowtests.yml
+++ b/buildspec-slowtests.yml
@@ -1,11 +1,15 @@
 version: 0.2
 
 phases:
+  pre_build:
+    commands:
+      - start-dockerd
+
   build:
     commands:
       - IGNORE_COVERAGE=-
 
       # slow tests
       - start_time=`date +%s`
-      - execute-command-if-has-matching-changes "tox -e py38 -- tests/integ -m slow_test -n 10 --durations 0" "tests/integ" "tests/data" "tests/conftest.py" "tests/__init__.py" "src/*.py" "setup.py" "setup.cfg" "buildspec-slowtests.yml"
+      - execute-command-if-has-matching-changes "tox -e py38 -- tests/integ -m slow_test -n 16 --durations 0" "tests/integ" "tests/data" "tests/conftest.py" "tests/__init__.py" "src/*.py" "setup.py" "setup.cfg" "buildspec-slowtests.yml"
       - ./ci-scripts/displaytime.sh 'py38 slow tests' $start_time
diff --git a/tests/data/multimodel/container/Dockerfile b/tests/data/multimodel/container/Dockerfile
index 1190057ef0..8290a6641a 100644
--- a/tests/data/multimodel/container/Dockerfile
+++ b/tests/data/multimodel/container/Dockerfile
@@ -1,4 +1,4 @@
-FROM 142577830533.dkr.ecr.us-east-2.amazonaws.com/ubuntu:16.04
+FROM public.ecr.aws/ubuntu/ubuntu:18.04
 
 # Set a docker label to advertise multi-model support on the container
 LABEL com.amazonaws.sagemaker.capabilities.multi-models=true
@@ -15,7 +15,7 @@ RUN apt-get update && \
     curl \
     vim \
     && rm -rf /var/lib/apt/lists/* \
-    && curl -O https://bootstrap.pypa.io/3.5/get-pip.py \
+    && curl -O https://bootstrap.pypa.io/get-pip.py \
     && python3 get-pip.py
 
 RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 1
diff --git a/tests/integ/test_lda.py b/tests/integ/test_lda.py
index a82ad7baec..6692370f1e 100644
--- a/tests/integ/test_lda.py
+++ b/tests/integ/test_lda.py
@@ -26,6 +26,7 @@
 from tests.integ.record_set import prepare_record_set_from_local_files
 
 
+@pytest.mark.slow_test
 @pytest.mark.skipif(
     tests.integ.test_region() in tests.integ.NO_LDA_REGIONS,
     reason="LDA image is not supported in certain regions",
diff --git a/tests/integ/test_model_quality_monitor.py b/tests/integ/test_model_quality_monitor.py
index fe41a40ab6..a1339fb19a 100644
--- a/tests/integ/test_model_quality_monitor.py
+++ b/tests/integ/test_model_quality_monitor.py
@@ -230,6 +230,7 @@ def test_model_quality_monitor(
     monitor.delete_monitoring_schedule()
 
 
+@pytest.mark.slow_test
 @pytest.mark.skipif(
     tests.integ.test_region() in tests.integ.NO_MODEL_MONITORING_REGIONS,
     reason="ModelMonitoring is not yet supported in this region.",
diff --git a/tests/integ/test_multidatamodel.py b/tests/integ/test_multidatamodel.py
index b8b7dec42f..6d1b60c9c8 100644
--- a/tests/integ/test_multidatamodel.py
+++ b/tests/integ/test_multidatamodel.py
@@ -22,8 +22,6 @@
 import pytest
 from botocore.exceptions import ClientError
 
-import tests
-
 from sagemaker import utils
 from sagemaker.amazon.randomcutforest import RandomCutForest
 from sagemaker.deserializers import StringDeserializer
@@ -53,10 +51,6 @@ def container_image(sagemaker_session):
 
     docker_client = docker.from_env()
 
-    # Base image pull
-    base_image = "142577830533.dkr.ecr.us-east-2.amazonaws.com/ubuntu:16.04"
-    docker_client.images.pull(base_image, auth_config={"username": username, "password": password})
-
     # Build and tag docker image locally
     image, build_log = docker_client.images.build(
         path=os.path.join(DATA_DIR, "multimodel", "container"),
@@ -134,10 +128,6 @@ def _ecr_login(ecr_client):
     return username, password
 
 
-@pytest.mark.skipif(
-    tests.integ.test_region() != "us-east-2",
-    reason="Pulling the base image is currently limited to us-east-2.",
-)
 def test_multi_data_model_deploy_pretrained_models(
     container_image, sagemaker_session, cpu_instance_type
 ):
@@ -198,10 +188,6 @@ def test_multi_data_model_deploy_pretrained_models(
 
 
 @pytest.mark.local_mode
-@pytest.mark.skipif(
-    tests.integ.test_region() != "us-east-2",
-    reason="Pulling the base image is currently limited to us-east-2.",
-)
 def test_multi_data_model_deploy_pretrained_models_local_mode(container_image, sagemaker_session):
     timestamp = sagemaker_timestamp()
     endpoint_name = "test-multimodel-endpoint-{}".format(timestamp)
@@ -262,11 +248,6 @@ def test_multi_data_model_deploy_pretrained_models_local_mode(container_image, s
         assert "Could not find endpoint" in str(exception.value)
 
 
-@pytest.mark.slow_test
-@pytest.mark.skipif(
-    tests.integ.test_region() != "us-east-2",
-    reason="Pulling the base image is currently limited to us-east-2.",
-)
 def test_multi_data_model_deploy_trained_model_from_framework_estimator(
     container_image,
     sagemaker_session,
@@ -381,10 +362,7 @@ def _mxnet_training_job(
         return mx.create_model(image_uri=container_image)
 
 
-@pytest.mark.skipif(
-    tests.integ.test_region() != "us-east-2",
-    reason="Pulling the base image is currently limited to us-east-2.",
-)
+@pytest.mark.slow_test
 def test_multi_data_model_deploy_train_model_from_amazon_first_party_estimator(
     container_image, sagemaker_session, cpu_instance_type
 ):
@@ -482,11 +460,6 @@ def __rcf_training_job(
         return rcf_model
 
 
-@pytest.mark.slow_test
-@pytest.mark.skipif(
-    tests.integ.test_region() != "us-east-2",
-    reason="Pulling the base image is currently limited to us-east-2.",
-)
 def test_multi_data_model_deploy_pretrained_models_update_endpoint(
     container_image, sagemaker_session, cpu_instance_type, alternative_cpu_instance_type
 ):
diff --git a/tests/integ/test_tf.py b/tests/integ/test_tf.py
index 62221c3223..ff76bc7d9d 100644
--- a/tests/integ/test_tf.py
+++ b/tests/integ/test_tf.py
@@ -155,6 +155,7 @@ def test_mnist_distributed(
     )
 
 
+@pytest.mark.slow_test
 def test_mnist_async(sagemaker_session, cpu_instance_type, tf_full_version, tf_full_py_version):
     estimator = TensorFlow(
         entry_point=SCRIPT,
diff --git a/tests/integ/test_tuner.py b/tests/integ/test_tuner.py
index b3287dfd3d..61356a8a09 100644
--- a/tests/integ/test_tuner.py
+++ b/tests/integ/test_tuner.py
@@ -402,6 +402,7 @@ def test_tuning_kmeans_identical_dataset_algorithm_tuner_from_non_terminal_paren
         )
 
 
+@pytest.mark.slow_test
 @pytest.mark.skipif(
     tests.integ.test_region() in tests.integ.NO_LDA_REGIONS,
     reason="LDA image is not supported in certain regions",