[pytorch][build]Upgrade python version in PT 1.6 containers (#860)

SergTogul · Sergey Togulev · tejaschumbalkar · web-flow · commit 5d4537a33efb · 2021-02-23T11:05:57.000-08:00
* Upgrade python version in PT 1.6 containers

* Fix path for inference 1.6 cu 101 image

* Installing python from conda-forge

* Updated pyyaml version.

Removed 39611 from ignored issues

* Installing ruamel-yaml to fix pip check's

conda 4.9.2 requires ruamel-yaml, which is not installed.

* Reformated test_safety_check.py

* Enabling all tests

* Skipping smprofiler test for pt 1.6

* Using DLG test 0.4.x since 0.5 may work only for pt 1.7+

* Using DLG test 0.4.x in eks since 0.5 may work only for pt 1.7+

* Add conditions for dgl test based on Pytorch version

* Fixed copy-paste error

* Fixed couple typos

* A couple updates after review

* Disabling safety check in PRs

* Rolling back buildspec.yml changes

* Rolling back config changes

Co-authored-by: Sergey Togulev &lt;togulev@amazon.com&gt;
Co-authored-by: Tejas Chumbalkar &lt;34728580+tejaschumbalkar@users.noreply.github.com&gt;
diff --git a/pytorch/inference/docker/1.6.0/py3/Dockerfile.cpu b/pytorch/inference/docker/1.6.0/py3/Dockerfile.cpu
@@ -5,7 +5,7 @@ LABEL dlc_major_version="1"
 LABEL com.amazonaws.sagemaker.capabilities.accept-bind-to-port=true
 LABEL com.amazonaws.sagemaker.capabilities.multi-models=true
 
-ARG PYTHON_VERSION=3.6.10
+ARG PYTHON_VERSION=3.6.13
 ARG OPEN_MPI_VERSION=4.0.1
 ARG TS_VERSION="0.2.1=py36_0"
 ARG PT_INFERENCE_URL=https://aws-pytorch-binaries.s3-us-west-2.amazonaws.com/r1.6.0_inference/20200727-223446/b0251e7e070e57f34ee08ac59ab4710081b41918/cpu/torch-1.6.0-cp36-cp36m-manylinux1_x86_64.whl
@@ -52,8 +52,9 @@ RUN curl -L -o ~/miniconda.sh https://repo.continuum.io/miniconda/Miniconda3-lat
  && ~/miniconda.sh -b -p /opt/conda \
  && rm ~/miniconda.sh \
  && /opt/conda/bin/conda update conda \
- && /opt/conda/bin/conda install -y \
+ && /opt/conda/bin/conda install -c conda-forge \
     python=$PYTHON_VERSION \
+ && /opt/conda/bin/conda install -y \
     cython==0.29.12 \
     ipython==7.7.0 \
     mkl-include==2019.4 \
@@ -123,8 +124,10 @@ RUN pip install --no-cache-dir "sagemaker-pytorch-inference>=2"
 
 RUN curl https://aws-dlc-licenses.s3.amazonaws.com/pytorch-1.6.0/license.txt -o /license.txt
 
-RUN conda install -y -c conda-forge pyyaml==5.3.1
-RUN pip install pillow==7.2.0 "awscli<2"
+RUN conda install -y -c conda-forge "pyyaml>5.4,<5.5"
+RUN pip install pillow==7.2.0 \
+    "awscli<2" \
+    ruamel-yaml 
 
 EXPOSE 8080 8081
 ENTRYPOINT ["python", "/usr/local/bin/dockerd-entrypoint.py"]
diff --git a/pytorch/inference/docker/1.6.0/py3/cu101/Dockerfile.gpu b/pytorch/inference/docker/1.6.0/py3/cu101/Dockerfile.gpu
@@ -6,7 +6,7 @@ LABEL dlc_major_version="1"
 LABEL com.amazonaws.sagemaker.capabilities.accept-bind-to-port=true
 
 # Add arguments to achieve the version, python and url
-ARG PYTHON_VERSION=3.6.10
+ARG PYTHON_VERSION=3.6.13
 ARG OPEN_MPI_VERSION=4.0.1
 ARG TS_VERSION="0.2.1=py36_0"
 ARG PT_INFERENCE_URL=https://aws-pytorch-binaries.s3-us-west-2.amazonaws.com/r1.6.0_inference/20200727-223446/b0251e7e070e57f34ee08ac59ab4710081b41918/gpu/torch-1.6.0-cp36-cp36m-manylinux1_x86_64.whl
@@ -79,8 +79,9 @@ RUN curl -L -o ~/miniconda.sh https://repo.continuum.io/miniconda/Miniconda3-lat
  && ~/miniconda.sh -b -p /opt/conda \
  && rm ~/miniconda.sh \
  && /opt/conda/bin/conda update conda \
- && /opt/conda/bin/conda install -y \
+ && /opt/conda/bin/conda install -c conda-forge \
     python=$PYTHON_VERSION \
+ && /opt/conda/bin/conda install -y \
     cython==0.29.12 \
     ipython==7.7.0 \
     mkl-include==2019.4 \
@@ -106,6 +107,7 @@ RUN conda install -c \
  && ln -s /opt/conda/bin/pip /usr/local/bin/pip3 \
  && pip install packaging==20.4 \
     enum-compat==0.0.3 \
+    ruamel-yaml \
  && conda install -y -c pytorch torchserve=$TS_VERSION \
  && conda install -y -c pytorch torch-model-archiver=$TS_VERSION
 
@@ -133,7 +135,7 @@ RUN pip install --no-cache-dir "sagemaker-pytorch-inference>=2"
 
 RUN curl https://aws-dlc-licenses.s3.amazonaws.com/pytorch-1.6.0/license.txt -o /license.txt
 
-RUN conda install -y -c conda-forge pyyaml==5.3.1
+RUN conda install -y -c conda-forge "pyyaml>5.4,<5.5"
 RUN pip install pillow==7.2.0 "awscli<2"
 
 EXPOSE 8080 8081
diff --git a/pytorch/training/docker/1.6.0/py3/Dockerfile.cpu b/pytorch/training/docker/1.6.0/py3/Dockerfile.cpu
@@ -4,7 +4,7 @@ LABEL maintainer="Amazon AI"
 LABEL dlc_major_version="1"
 
 # Add arguments to achieve the version, python and url
-ARG PYTHON_VERSION=3.6.10
+ARG PYTHON_VERSION=3.6.13
 ARG OPEN_MPI_VERSION=4.0.1
 
 # The smdebug pipeline relies for following format to perform string replace and trigger DLC pipeline for validating
@@ -73,8 +73,9 @@ RUN curl -L -o ~/miniconda.sh https://repo.continuum.io/miniconda/Miniconda3-lat
  && chmod +x ~/miniconda.sh \
  && ~/miniconda.sh -b -p /opt/conda \
  && rm ~/miniconda.sh \
- && /opt/conda/bin/conda install -y -c anaconda \
+ && /opt/conda/bin/conda install -c conda-forge \
     python=$PYTHON_VERSION \
+ && /opt/conda/bin/conda install -y -c anaconda \
     numpy \
     ipython \
     mkl \
@@ -102,6 +103,7 @@ RUN pip install --upgrade pip --trusted-host pypi.org --trusted-host files.pytho
     fastai==1.0.61 \
     scipy==1.2.2 \
     click \
+    ruamel-yaml \
     "cryptography>3.2" \
     smdebug==${SMDEBUG_VERSION} \
     smclarify \
diff --git a/pytorch/training/docker/1.6.0/py3/cu101/Dockerfile.gpu b/pytorch/training/docker/1.6.0/py3/cu101/Dockerfile.gpu
@@ -4,7 +4,7 @@ FROM nvidia/cuda:10.1-cudnn7-devel-ubuntu16.04
 LABEL maintainer="Amazon AI"
 LABEL dlc_major_version="1"
 
-ARG PYTHON_VERSION=3.6.10
+ARG PYTHON_VERSION=3.6.13
 ARG OPEN_MPI_VERSION=4.0.1
 ARG CUBLAS_VERSION=10.2.1.243-1_amd64
 ARG OPEN_MPI_PATH=/home/.openmpi
@@ -102,8 +102,9 @@ RUN ompi_info --parsable --all | grep mpi_built_with_cuda_support:value \
  && chmod +x ~/miniconda.sh \
  && ~/miniconda.sh -b -p /opt/conda \
  && rm ~/miniconda.sh \
- && /opt/conda/bin/conda install -y -c anaconda \
+ && /opt/conda/bin/conda install -c conda-forge \
     python=$PYTHON_VERSION \
+ && /opt/conda/bin/conda install -y -c anaconda \
     numpy \
     ipython \
     mkl \
@@ -149,6 +150,7 @@ RUN pip install \
     Pillow \
     scipy \
     click \
+    ruamel-yaml \
     mpi4py==3.0.3 \
     cmake==3.18.2.post1 \
  && pip install --no-cache-dir -U ${PT_TRAINING_URL} \
diff --git a/pytorch/training/docker/1.6.0/py3/cu110/Dockerfile.gpu b/pytorch/training/docker/1.6.0/py3/cu110/Dockerfile.gpu
@@ -4,7 +4,7 @@ FROM nvidia/cuda:11.0-cudnn8-devel-ubuntu18.04
 LABEL maintainer="Amazon AI"
 LABEL dlc_major_version="3"
 
-ARG PYTHON_VERSION=3.6.10
+ARG PYTHON_VERSION=3.6.13
 ARG OPEN_MPI_VERSION=4.0.1
 ARG CUBLAS_VERSION=11.2.0.252-1_amd64
 ARG OPEN_MPI_PATH=/home/.openmpi
@@ -103,8 +103,9 @@ RUN ompi_info --parsable --all | grep mpi_built_with_cuda_support:value \
  && chmod +x ~/miniconda.sh \
  && ~/miniconda.sh -b -p ${CONDA_PREFIX} \
  && rm ~/miniconda.sh \
- && ${CONDA_PREFIX}/bin/conda install -y -c anaconda \
+ && ${CONDA_PREFIX}/bin/conda install -c conda-forge \
     python=$PYTHON_VERSION \
+ && ${CONDA_PREFIX}/bin/conda install -y -c anaconda \
     numpy \
     ipython \
     mkl \
@@ -161,6 +162,7 @@ RUN pip install --no-cache-dir -U \
     scipy \
     pybind11 \
     click \
+    ruamel-yaml \
     mpi4py==3.0.3 \
     cmake==3.18.2.post1 \
     torchnet \
diff --git a/test/dlc_tests/conftest.py b/test/dlc_tests/conftest.py
@@ -17,10 +17,10 @@
 
 from test import test_utils
 from test.test_utils import (
-    is_benchmark_dev_context, get_framework_and_version_from_tag, get_job_type_from_image, is_tf_version, 
-    is_below_tf_version, is_below_mxnet_version, 
+    is_benchmark_dev_context, get_framework_and_version_from_tag, get_job_type_from_image, is_tf_version,
+    is_below_tf_version, is_below_mxnet_version, is_below_pytorch_version, 
     DEFAULT_REGION, P3DN_REGION, UBUNTU_18_BASE_DLAMI_US_EAST_1, UBUNTU_18_BASE_DLAMI_US_WEST_2,
-    PT_GPU_PY3_BENCHMARK_IMAGENET_AMI_US_EAST_1, KEYS_TO_DESTROY_FILE
+    PT_GPU_PY3_BENCHMARK_IMAGENET_AMI_US_EAST_1, KEYS_TO_DESTROY_FILE 
 )
 from test.test_utils.test_reporting import TestReportGenerator
 
@@ -327,6 +327,11 @@ def mx18_and_above_only():
     pass
 
 
+@pytest.fixture(scope="session")
+def pt17_and_above_only():
+    pass
+
+
 def framework_version_within_limit(metafunc_obj, image):
     """
     Test all pytest fixtures for TensorFlow version limits, and return True if all requirements are satisfied
@@ -347,6 +352,10 @@ def framework_version_within_limit(metafunc_obj, image):
         mx18_requirement_failed = "mx18_and_above_only" in metafunc_obj.fixturenames and is_below_mxnet_version("1.8", image)
         if mx18_requirement_failed :
             return False
+    if image_framework_name == "pytorch" :
+        pt17_requirement_failed = "pt17_and_above_only" in metafunc_obj.fixturenames and is_below_pytorch_version("1.7", image)
+        if pt17_requirement_failed :
+            return False
     return True
 
 
diff --git a/test/dlc_tests/container_tests/bin/dgl_tests/testPyTorchDGL b/test/dlc_tests/container_tests/bin/dgl_tests/testPyTorchDGL
@@ -4,11 +4,16 @@ HOME_DIR=/test
 BIN_DIR=${HOME_DIR}/bin
 LOG_DIR=${HOME_DIR}/logs
 
+OLD_DGL_VERSION=0.4.x
+NEW_DGL_VERSION=0.5.x
+
 DGL_RELEASE_TAG=$(python -c "import dgl; dgl_versions = dgl.__version__.split('.'); print(f'{dgl_versions[0]}.{dgl_versions[1]}.x')")
 
-# hard coded test files from 0.5.x branch. Change it back once 0.6.x is released
-# git clone -b ${DGL_RELEASE_TAG} https://github.com/dmlc/dgl.git ${HOME_DIR}/artifacts/dgl
-git clone -b 0.5.x https://github.com/dmlc/dgl.git ${HOME_DIR}/artifacts/dgl
+if [[ ${DGL_RELEASE_TAG} == ${OLD_DGL_VERSION} ]]; then
+    git clone -b ${OLD_DGL_VERSION} https://github.com/dmlc/dgl.git ${HOME_DIR}/artifacts/dgl 
+else
+  git clone -b ${NEW_DGL_VERSION} https://github.com/dmlc/dgl.git ${HOME_DIR}/artifacts/dgl
+fi 
 ${BIN_DIR}/dgl_tests/testDGLHelper python pytorch || exit 1
 
 exit 0
diff --git a/test/dlc_tests/ec2/test_smdebug.py b/test/dlc_tests/ec2/test_smdebug.py
@@ -41,7 +41,7 @@ def test_smdebug_gpu(training, ec2_connection, region, ec2_instance_type, gpu_on
 @pytest.mark.model("mnist")
 @pytest.mark.parametrize("ec2_instance_type", SMDEBUG_EC2_GPU_INSTANCE_TYPE, indirect=True)
 @pytest.mark.flaky(reruns=0)
-def test_smprofiler_gpu(training, ec2_connection, region, ec2_instance_type, gpu_only, py3_only, tf23_and_above_only):
+def test_smprofiler_gpu(training, ec2_connection, region, ec2_instance_type, gpu_only, py3_only, tf23_and_above_only, pt17_and_above_only):
     # Running the profiler tests for pytorch and tensorflow2 frameworks only.
     # This code needs to be modified past reInvent 2020
     framework = get_framework_from_image_uri(training)
diff --git a/test/dlc_tests/eks/pytorch/training/test_eks_pytorch_training.py b/test/dlc_tests/eks/pytorch/training/test_eks_pytorch_training.py
@@ -10,7 +10,7 @@
 from retrying import retry
 
 import test.test_utils.eks as eks_utils
-from test.test_utils import is_pr_context, SKIP_PR_REASON
+from test.test_utils import is_pr_context, SKIP_PR_REASON, is_below_pytorch_version
 from test.test_utils import get_framework_and_version_from_tag, get_cuda_version_from_tag
 from packaging.version import Version
 
@@ -89,7 +89,10 @@ def test_eks_pytorch_dgl_single_node_training(pytorch_training, py3_only):
     yaml_path = os.path.join(os.sep, "tmp", f"pytorch_single_node_training_dgl_{rand_int}.yaml")
     pod_name = f"pytorch-single-node-training-dgl-{rand_int}"
 
-    dgl_branch = "0.5.x"
+    if is_below_pytorch_version("1.7", pytorch_training):
+        dgl_branch = "0.4.x"
+    else:
+        dgl_branch = "0.5.x"
 
     args = (
         f"git clone -b {dgl_branch} https://github.com/dmlc/dgl.git && "
diff --git a/test/dlc_tests/sanity/test_safety_check.py b/test/dlc_tests/sanity/test_safety_check.py
@@ -28,25 +28,20 @@
             # 38449, 38450, 38451, 38452: for shipping pillow<=6.2.2 - the last available version for py2
             # 35015: for shipping pycrypto<=2.6.1 - the last available version for py2
             "py2": ['38449', '38450', '38451', '38452', '35015'],
-            # for shipping pyyaml v5.3.1 - blocked on upgrading to v5.4.1 due to dependency on awscli
-            "py3": ['39611']
+            "py3": []
         },
         "inference": {
             # for shipping pillow<=6.2.2 - the last available version for py2
             "py2": ['38449', '38450', '38451', '38452'],
-            # for shipping pyyaml v5.3.1 - blocked on upgrading to v5.4.1 due to dependency on awscli
-            "py3": ['39611']
+            "py3": []
         },
         "inference-eia": {
             # for shipping pillow<=6.2.2 - the last available version for py2
             "py2": ['38449', '38450', '38451', '38452'],
-            # for shipping pyyaml v5.3.1 - blocked on upgrading to v5.4.1 due to dependency on awscli
-            "py3": ['39611']
+            "py3": []
         },
         "inference-neuron": {
             "py3": [
-                # for shipping pyyaml v5.3.1 - blocked on upgrading to v5.4.1 due to dependency on awscli
-                '39611',
                 # 39409, 39408, 39407, 39406: TF 1.15.5 is on par with TF 2.0.4, 2.1.3, 2.2.2, 2.3.2 in security patches
                 '39409', '39408', '39407', '39406',
             ],
@@ -58,24 +53,20 @@
             "py2": ['36810',
                     # for shipping pillow<=6.2.2 - the last available version for py2
                     '38449', '38450', '38451', '38452'],
-            # for shipping pyyaml v5.3.1 - blocked on upgrading to v5.4.1 due to dependency on awscli
-            "py3": ['39611']
+            "py3": []
         },
         "inference": {
             # for shipping pillow<=6.2.2 - the last available version for py2
             "py2": ['38449', '38450', '38451', '38452'],
-            # for shipping pyyaml v5.3.1 - blocked on upgrading to v5.4.1 due to dependency on awscli
-            "py3": ['39611']
+            "py3": []
         },
         "training": {
             # for shipping pillow<=6.2.2 - the last available version for py2
             "py2": ['38449', '38450', '38451', '38452'],
-            # for shipping pyyaml v5.3.1 - blocked on upgrading to v5.4.1 due to dependency on awscli
-            "py3": ['39611']
+            "py3": []
         },
-        "inference-neuron":{
-            # for shipping pyyaml v5.3.1 - blocked on upgrading to v5.4.1 due to dependency on awscli
-            "py3": ['39611']
+        "inference-neuron": {
+            "py3": []
         }
     },
     "pytorch": {
@@ -92,9 +83,8 @@
         "inference-eia": {
             "py3": []
         },
-        "inference-neuron":{
-            # for shipping pyyaml v5.3.1 - blocked on upgrading to v5.4.1 due to dependency on awscli
-            "py3": ['39611']
+        "inference-neuron": {
+            "py3": []
         }
     }
 }
@@ -135,7 +125,7 @@ def _get_latest_package_version(package):
 @pytest.mark.model("N/A")
 @pytest.mark.skipif(not is_dlc_cicd_context(), reason="Skipping test because it is not running in dlc cicd infra")
 @pytest.mark.skipif(not is_mainline_context(),
-                    reason="Skipping the test to decrease the number of calls to the Safety Check DB. "
+                   reason="Skipping the test to decrease the number of calls to the Safety Check DB. "
                            "Test will be executed in the 'mainline' pipeline only")
 def test_safety(image):
     """
diff --git a/test/test_utils/__init__.py b/test/test_utils/__init__.py
@@ -112,6 +112,19 @@ def is_below_mxnet_version(version_upper_bound, image_uri):
     return image_framework_name == "mxnet" and image_framework_version in required_version_specifier_set
 
 
+def is_below_pytorch_version(version_upper_bound, image_uri):
+    """
+    Validate that image_uri has framework version strictly less than version_upper_bound
+
+    :param version_upper_bound: str Framework version that image_uri is required to be below
+    :param image_uri: str ECR Image URI for the image to be validated
+    :return: bool True if image_uri has framework version less than version_upper_bound, else False
+    """
+    image_framework_name, image_framework_version = get_framework_and_version_from_tag(image_uri)
+    required_version_specifier_set = SpecifierSet(f"<{version_upper_bound}")
+    return image_framework_name == "pytorch" and image_framework_version in required_version_specifier_set
+
+
 def get_repository_local_path():
     git_repo_path = os.getcwd().split("/test/")[0]
     return git_repo_path
