Skip to content

Commit 9880409

Browse files
saimidusaimidu
authored
[tensorflow][build] Upgrade py version on TF 2.4.1 dockerfiles (#876)
1 parent 499a810 commit 9880409

File tree

8 files changed

+158
-60
lines changed

8 files changed

+158
-60
lines changed

‎tensorflow/buildspec-2-4.yml

Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,108 @@
1+
account_id: &ACCOUNT_ID <set-$ACCOUNT_ID-in-environment>
2+
region: &REGION <set-$REGION-in-environment>
3+
framework: &FRAMEWORK tensorflow
4+
version: &VERSION 2.4.1
5+
short_version: &SHORT_VERSION 2.4
6+
7+
repository_info:
8+
training_repository: &TRAINING_REPOSITORY
9+
image_type: &TRAINING_IMAGE_TYPE training
10+
root: !join [ *FRAMEWORK, "/", *TRAINING_IMAGE_TYPE ]
11+
repository_name: &REPOSITORY_NAME !join [pr, "-", *FRAMEWORK, "-", *TRAINING_IMAGE_TYPE]
12+
repository: &REPOSITORY !join [ *ACCOUNT_ID, .dkr.ecr., *REGION, .amazonaws.com/,
13+
*REPOSITORY_NAME ]
14+
inference_repository: &INFERENCE_REPOSITORY
15+
image_type: &INFERENCE_IMAGE_TYPE inference
16+
root: !join [ *FRAMEWORK, "/", *INFERENCE_IMAGE_TYPE ]
17+
repository_name: &REPOSITORY_NAME !join [pr, "-", *FRAMEWORK, "-", *INFERENCE_IMAGE_TYPE]
18+
repository: &REPOSITORY !join [ *ACCOUNT_ID, .dkr.ecr., *REGION, .amazonaws.com/,
19+
*REPOSITORY_NAME ]
20+
21+
context:
22+
training_context: &TRAINING_CONTEXT
23+
dockerd-entrypoint:
24+
source: docker/build_artifacts/dockerd-entrypoint.py
25+
target: dockerd-entrypoint.py
26+
inference_context: &INFERENCE_CONTEXT
27+
sagemaker_package_name:
28+
source: docker/build_artifacts/sagemaker
29+
target: sagemaker
30+
init:
31+
source: docker/build_artifacts/__init__.py
32+
target: __init__.py
33+
dockerd-entrypoint:
34+
source: docker/build_artifacts/dockerd-entrypoint.py
35+
target: dockerd-entrypoint.py
36+
37+
images:
38+
BuildTensorflowCpuPy37TrainingDockerImage:
39+
<<: *TRAINING_REPOSITORY
40+
build: &TENSORFLOW_CPU_TRAINING_PY3 false
41+
image_size_baseline: &IMAGE_SIZE_BASELINE 4489
42+
device_type: &DEVICE_TYPE cpu
43+
python_version: &DOCKER_PYTHON_VERSION py3
44+
tag_python_version: &TAG_PYTHON_VERSION py37
45+
os_version: &OS_VERSION ubuntu18.04
46+
tag: !join [ *VERSION, "-", *DEVICE_TYPE, "-", *TAG_PYTHON_VERSION, "-", *OS_VERSION
47+
]
48+
docker_file: !join [ docker/, *SHORT_VERSION, /, *DOCKER_PYTHON_VERSION, /Dockerfile.,
49+
*DEVICE_TYPE ]
50+
context:
51+
<<: *TRAINING_CONTEXT
52+
BuildTensorflowGpuPy37Cu110TrainingDockerImage:
53+
<<: *TRAINING_REPOSITORY
54+
build: &TENSORFLOW_GPU_TRAINING_PY3 false
55+
image_size_baseline: &IMAGE_SIZE_BASELINE 7738
56+
device_type: &DEVICE_TYPE gpu
57+
python_version: &DOCKER_PYTHON_VERSION py3
58+
tag_python_version: &TAG_PYTHON_VERSION py37
59+
cuda_version: &CUDA_VERSION cu110
60+
os_version: &OS_VERSION ubuntu18.04
61+
tag: !join [ *VERSION, "-", *DEVICE_TYPE, "-", *TAG_PYTHON_VERSION, "-", *CUDA_VERSION,
62+
"-", *OS_VERSION ]
63+
docker_file: !join [ docker/, *SHORT_VERSION, /, *DOCKER_PYTHON_VERSION, /, *CUDA_VERSION,
64+
/Dockerfile., *DEVICE_TYPE ]
65+
context:
66+
<<: *TRAINING_CONTEXT
67+
BuildTensorflowExampleGpuPy37Cu110TrainingDockerImage:
68+
<<: *TRAINING_REPOSITORY
69+
build: &TENSORFLOW_GPU_TRAINING_PY3 false
70+
image_size_baseline: &IMAGE_SIZE_BASELINE 7738
71+
base_image_name: BuildTensorflowGpuPy37Cu110TrainingDockerImage
72+
device_type: &DEVICE_TYPE gpu
73+
python_version: &DOCKER_PYTHON_VERSION py3
74+
tag_python_version: &TAG_PYTHON_VERSION py37
75+
cuda_version: &CUDA_VERSION cu110
76+
os_version: &OS_VERSION ubuntu18.04
77+
tag: !join [ *VERSION, "-", *DEVICE_TYPE, "-", *TAG_PYTHON_VERSION, "-", *CUDA_VERSION,
78+
"-", *OS_VERSION, "-example" ]
79+
docker_file: !join [ docker/, *SHORT_VERSION, /, *DOCKER_PYTHON_VERSION, /example,
80+
/Dockerfile., *DEVICE_TYPE ]
81+
context:
82+
<<: *TRAINING_CONTEXT
83+
BuildTensorflowCPUInferencePy3DockerImage:
84+
<<: *INFERENCE_REPOSITORY
85+
build: &TENSORFLOW_CPU_INFERENCE_PY3 false
86+
image_size_baseline: 4899
87+
device_type: &DEVICE_TYPE cpu
88+
python_version: &DOCKER_PYTHON_VERSION py3
89+
tag_python_version: &TAG_PYTHON_VERSION py37
90+
cuda_version: &CUDA_VERSION cu110
91+
os_version: &OS_VERSION ubuntu18.04
92+
tag: !join [ *VERSION, "-", *DEVICE_TYPE, "-", *TAG_PYTHON_VERSION, "-", *OS_VERSION ]
93+
docker_file: !join [ docker/, *SHORT_VERSION, /, *DOCKER_PYTHON_VERSION, /Dockerfile., *DEVICE_TYPE ]
94+
context:
95+
<<: *INFERENCE_CONTEXT
96+
BuildTensorflowGPUInferencePy3DockerImage:
97+
<<: *INFERENCE_REPOSITORY
98+
build: &TENSORFLOW_GPU_INFERENCE_PY3 false
99+
image_size_baseline: 7738
100+
device_type: &DEVICE_TYPE gpu
101+
python_version: &DOCKER_PYTHON_VERSION py3
102+
tag_python_version: &TAG_PYTHON_VERSION py37
103+
cuda_version: &CUDA_VERSION cu110
104+
os_version: &OS_VERSION ubuntu18.04
105+
tag: !join [ *VERSION, "-", *DEVICE_TYPE, "-", *TAG_PYTHON_VERSION, "-", *CUDA_VERSION, "-", *OS_VERSION ]
106+
docker_file: !join [ docker/, *SHORT_VERSION, /, *DOCKER_PYTHON_VERSION, /, *CUDA_VERSION, /Dockerfile., *DEVICE_TYPE ]
107+
context:
108+
<<: *INFERENCE_CONTEXT

‎tensorflow/buildspec.yml

Lines changed: 28 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ repository_info:
1111
repository_name: &REPOSITORY_NAME !join [pr, "-", *FRAMEWORK, "-", *TRAINING_IMAGE_TYPE]
1212
repository: &REPOSITORY !join [ *ACCOUNT_ID, .dkr.ecr., *REGION, .amazonaws.com/,
1313
*REPOSITORY_NAME ]
14-
inference_repository:
14+
inference_repository: &INFERENCE_REPOSITORY
1515
image_type: &INFERENCE_IMAGE_TYPE inference
1616
root: !join [ *FRAMEWORK, "/", *INFERENCE_IMAGE_TYPE ]
1717
repository_name: &REPOSITORY_NAME !join [pr, "-", *FRAMEWORK, "-", *INFERENCE_IMAGE_TYPE]
@@ -23,7 +23,7 @@ context:
2323
dockerd-entrypoint:
2424
source: docker/build_artifacts/dockerd-entrypoint.py
2525
target: dockerd-entrypoint.py
26-
inference_context:
26+
inference_context: &INFERENCE_CONTEXT
2727
sagemaker_package_name:
2828
source: docker/build_artifacts/sagemaker
2929
target: sagemaker
@@ -80,3 +80,29 @@ images:
8080
/Dockerfile., *DEVICE_TYPE ]
8181
context:
8282
<<: *TRAINING_CONTEXT
83+
BuildTensorflowCPUInferencePy3DockerImage:
84+
<<: *INFERENCE_REPOSITORY
85+
build: &TENSORFLOW_CPU_INFERENCE_PY3 false
86+
image_size_baseline: 4899
87+
device_type: &DEVICE_TYPE cpu
88+
python_version: &DOCKER_PYTHON_VERSION py3
89+
tag_python_version: &TAG_PYTHON_VERSION py37
90+
cuda_version: &CUDA_VERSION cu110
91+
os_version: &OS_VERSION ubuntu18.04
92+
tag: !join [ *VERSION, "-", *DEVICE_TYPE, "-", *TAG_PYTHON_VERSION, "-", *OS_VERSION ]
93+
docker_file: !join [ docker/, *SHORT_VERSION, /, *DOCKER_PYTHON_VERSION, /Dockerfile., *DEVICE_TYPE ]
94+
context:
95+
<<: *INFERENCE_CONTEXT
96+
BuildTensorflowGPUInferencePy3DockerImage:
97+
<<: *INFERENCE_REPOSITORY
98+
build: &TENSORFLOW_GPU_INFERENCE_PY3 false
99+
image_size_baseline: 7738
100+
device_type: &DEVICE_TYPE gpu
101+
python_version: &DOCKER_PYTHON_VERSION py3
102+
tag_python_version: &TAG_PYTHON_VERSION py37
103+
cuda_version: &CUDA_VERSION cu110
104+
os_version: &OS_VERSION ubuntu18.04
105+
tag: !join [ *VERSION, "-", *DEVICE_TYPE, "-", *TAG_PYTHON_VERSION, "-", *CUDA_VERSION, "-", *OS_VERSION ]
106+
docker_file: !join [ docker/, *SHORT_VERSION, /, *DOCKER_PYTHON_VERSION, /, *CUDA_VERSION, /Dockerfile., *DEVICE_TYPE ]
107+
context:
108+
<<: *INFERENCE_CONTEXT

‎tensorflow/inference/docker/2.4/py3/Dockerfile.cpu

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ LABEL com.amazonaws.sagemaker.capabilities.multi-models=true
1010
ARG PYTHON=python3.7
1111
ARG PYTHON_PIP=python3-pip
1212
ARG PIP=pip3
13-
ARG PYTHON_VERSION=3.7.7
13+
ARG PYTHON_VERSION=3.7.10
1414
ARG TFS_SHORT_VERSION=2.4
1515
ARG TFS_URL=https://aws-tensorflow-binaries.s3-us-west-2.amazonaws.com/serving/r2.4_aws/20210125-223221/cpu/py37/tensorflow_model_server
1616

@@ -65,7 +65,7 @@ RUN wget https://www.python.org/ftp/python/$PYTHON_VERSION/Python-$PYTHON_VERSIO
6565
&& tar -xvf Python-$PYTHON_VERSION.tgz \
6666
&& cd Python-$PYTHON_VERSION \
6767
&& ./configure && make && make install \
68-
&& make && make install && rm -rf ../Python-$PYTHON_VERSION*
68+
&& rm -rf ../Python-$PYTHON_VERSION*
6969

7070
RUN ${PIP} --no-cache-dir install --upgrade \
7171
pip \
@@ -117,4 +117,3 @@ RUN chmod +x /usr/local/bin/deep_learning_container.py
117117
RUN curl https://aws-dlc-licenses.s3.amazonaws.com/tensorflow-2.4/license.txt -o /license.txt
118118

119119
CMD ["/usr/bin/tf_serving_entrypoint.sh"]
120-

‎tensorflow/inference/docker/2.4/py3/cu110/Dockerfile.gpu

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ LABEL com.amazonaws.sagemaker.capabilities.accept-bind-to-port=true
99
ARG PYTHON=python3.7
1010
ARG PYTHON_PIP=python3-pip
1111
ARG PIP=pip3
12-
ARG PYTHON_VERSION=3.7.7
12+
ARG PYTHON_VERSION=3.7.10
1313

1414
ARG TFS_SHORT_VERSION=2.4
1515
ARG TFS_URL=https://aws-tensorflow-binaries.s3-us-west-2.amazonaws.com/serving/r2.4_aws/20210125-223221/gpu/py37/tensorflow_model_server
@@ -78,7 +78,7 @@ RUN wget https://www.python.org/ftp/python/$PYTHON_VERSION/Python-$PYTHON_VERSIO
7878
&& tar -xvf Python-$PYTHON_VERSION.tgz \
7979
&& cd Python-$PYTHON_VERSION \
8080
&& ./configure && make && make install \
81-
&& make && make install && rm -rf ../Python-$PYTHON_VERSION*
81+
&& rm -rf ../Python-$PYTHON_VERSION*
8282

8383
RUN ${PIP} --no-cache-dir install --upgrade \
8484
pip \
@@ -156,4 +156,3 @@ RUN chmod +x /usr/local/bin/deep_learning_container.py
156156
RUN curl https://aws-dlc-licenses.s3.amazonaws.com/tensorflow-2.4/license.txt -o /license.txt
157157

158158
CMD ["/usr/bin/tf_serving_entrypoint.sh"]
159-

‎tensorflow/training/docker/2.4/py3/Dockerfile.cpu

Lines changed: 8 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -25,8 +25,7 @@ ENV LC_ALL=C.UTF-8
2525
ARG PYTHON=python3.7
2626
ARG PYTHON_PIP=python3-pip
2727
ARG PIP=pip3
28-
ARG PYTHON_VERSION=3.7.7
29-
ARG OPENSSL_VERSION=1.1.1g
28+
ARG PYTHON_VERSION=3.7.10
3029

3130
ARG TF_URL=https://aws-tensorflow-binaries.s3-us-west-2.amazonaws.com/tensorflow/r2.4_aws/20210127-150238/cpu/py37/tensorflow_cpu-2.4.1-cp37-cp37m-manylinux2010_x86_64.whl
3231

@@ -36,7 +35,8 @@ ARG ESTIMATOR_URL=https://aws-tensorflow-binaries.s3-us-west-2.amazonaws.com/est
3635
# the nightly builds. Therefore, while updating the smdebug version, please ensure that the format is not disturbed.
3736
ARG SMDEBUG_VERSION=1.0.4
3837

39-
RUN apt-get update && apt-get install -y --no-install-recommends \
38+
RUN apt-get update \
39+
&& apt-get install -y --no-install-recommends \
4040
build-essential \
4141
openssh-client \
4242
openssh-server \
@@ -45,6 +45,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
4545
emacs \
4646
git \
4747
libtemplate-perl \
48+
libssl1.1 \
49+
openssl \
4850
wget \
4951
vim \
5052
zlib1g-dev \
@@ -113,39 +115,22 @@ RUN ${PIP} --no-cache-dir install --upgrade \
113115
pip \
114116
setuptools
115117

116-
RUN wget -c https://www.openssl.org/source/openssl-${OPENSSL_VERSION}.tar.gz \
117-
&& apt remove -y --purge openssl \
118-
&& rm -rf /usr/include/openssl \
119-
&& apt-get update \
120-
&& apt-get install -y \
121-
ca-certificates \
122-
&& tar -xzvf openssl-${OPENSSL_VERSION}.tar.gz \
123-
&& cd openssl-${OPENSSL_VERSION} \
124-
&& ./config && make && make test \
125-
&& make install \
126-
&& ldconfig \
127-
&& cd .. && rm -rf openssl-*
128-
129-
# when we remove previous openssl, the ca-certificates pkgs and its symlinks gets deleted
130-
# causing sslcertverificationerror the below steps are to fix that
131-
RUN ln -s /etc/ssl/certs/*.* /usr/local/ssl/certs/
132-
133118
# Some TF tools expect a "python" binary
134119
RUN ln -s $(which ${PYTHON}) /usr/local/bin/python \
135120
&& ln -s $(which ${PIP}) /usr/bin/pip
136121

137122
RUN apt-get update && apt-get -y install cmake protobuf-compiler
138123

139-
# install PyYAML==5.3.1 to avoid conflict with latest awscli
140-
# # python-dateutil==2.8.1 to satisfy botocore associated with latest awscli
141124
RUN ${PIP} install --no-cache-dir -U \
142125
numpy==1.19.1 \
143126
scipy==1.5.2 \
144127
scikit-learn==0.23 \
145128
pandas==1.1 \
146129
Pillow==7.2.0 \
130+
# python-dateutil==2.8.1 to satisfy botocore associated with latest awscli
147131
python-dateutil==2.8.1 \
148-
pyYAML==5.3.1 \
132+
# install PyYAML>=5.4 to avoid conflict with latest awscli
133+
"pyYAML>=5.4,<5.5" \
149134
requests==2.24.0 \
150135
"awscli<2" \
151136
mpi4py==3.0.3 \

‎tensorflow/training/docker/2.4/py3/cu110/Dockerfile.gpu

Lines changed: 6 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -25,8 +25,7 @@ ENV MANUAL_BUILD=0
2525
ARG PYTHON=python3.7
2626
ARG PYTHON_PIP=python3-pip
2727
ARG PIP=pip3
28-
ARG PYTHON_VERSION=3.7.7
29-
ARG OPENSSL_VERSION=1.1.1i
28+
ARG PYTHON_VERSION=3.7.10
3029
ARG OPEN_MPI_PATH=/usr/local
3130

3231
ARG TF_URL=https://aws-tensorflow-binaries.s3-us-west-2.amazonaws.com/tensorflow/r2.4_aws/20210127-150238/gpu/py37/cu110/tensorflow_gpu-2.4.1-cp37-cp37m-manylinux2010_x86_64.whl
@@ -66,6 +65,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends --allow-unauthe
6665
wget \
6766
libtool \
6867
vim \
68+
libssl1.1 \
69+
openssl \
6970
build-essential \
7071
openssh-client \
7172
openssh-server \
@@ -172,29 +173,10 @@ RUN ${PIP} --no-cache-dir install --upgrade \
172173
pip \
173174
setuptools
174175

175-
RUN wget -c https://www.openssl.org/source/openssl-${OPENSSL_VERSION}.tar.gz \
176-
&& apt-get update \
177-
&& apt remove -y --purge openssl \
178-
&& rm -rf /usr/include/openssl \
179-
&& apt-get install -y \
180-
ca-certificates \
181-
&& tar -xzvf openssl-${OPENSSL_VERSION}.tar.gz \
182-
&& cd openssl-${OPENSSL_VERSION} \
183-
&& ./config && make && make test \
184-
&& make install \
185-
&& ldconfig \
186-
&& cd .. && rm -rf openssl-*
187-
188-
# when we remove previous openssl, the ca-certificates pkgs and its symlinks gets deleted
189-
# causing sslcertverificationerror the below steps are to fix that
190-
RUN ln -s /etc/ssl/certs/*.* /usr/local/ssl/certs/
191-
192176
# Some TF tools expect a "python" binary
193177
RUN ln -s $(which ${PYTHON}) /usr/local/bin/python \
194178
&& ln -s $(which ${PIP}) /usr/bin/pip
195179

196-
# install PyYAML==5.3.1 to avoid conflict with latest awscli
197-
# # python-dateutil==2.8.1 to satisfy botocore associated with latest awscli
198180
RUN ${PIP} install --no-cache-dir -U \
199181
pybind11 \
200182
cmake==3.18.2.post1 \
@@ -203,8 +185,10 @@ RUN ${PIP} install --no-cache-dir -U \
203185
scikit-learn==0.23 \
204186
pandas==1.1 \
205187
Pillow==7.2.0 \
188+
# python-dateutil==2.8.1 to satisfy botocore associated with latest awscli
206189
python-dateutil==2.8.1 \
207-
pyYAML==5.3.1 \
190+
# install PyYAML>=5.4 to avoid conflict with latest awscli
191+
"pyYAML>=5.4,<5.5" \
208192
requests==2.24.0 \
209193
"awscli<2" \
210194
mpi4py==3.0.3 \

‎test/dlc_tests/sanity/test_pre_release.py

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -195,10 +195,7 @@ class DependencyCheckFailure(Exception):
195195

196196
def _run_dependency_check_test(image, ec2_connection, processor):
197197
# Record any whitelisted medium/low severity CVEs; I.E. allowed_vulnerabilities = {CVE-1000-5555, CVE-9999-9999}
198-
allowed_vulnerabilities = {
199-
# Those vulnerabilities are fixed. Current openssl version is 1.1.1g. These are false positive
200-
'CVE-2016-2109', 'CVE-2016-2177', 'CVE-2016-6303', 'CVE-2016-2182'
201-
}
198+
allowed_vulnerabilities = set()
202199

203200
container_name = f"dep_check_{processor}"
204201
report_addon = get_container_name("depcheck-report", image)
@@ -239,8 +236,8 @@ def _run_dependency_check_test(image, ec2_connection, processor):
239236
return
240237

241238
raise DependencyCheckFailure(
242-
f"Unrecognized CVES have been reported : {vulnerability_severity}. "
243-
f"Allowed vulnerabilites are {allowed_vulnerabilities or None}. Please see "
239+
f"Unrecognized CVEs have been reported : {vulnerability_severity}. "
240+
f"Allowed vulnerabilities are {allowed_vulnerabilities or None}. Please see "
244241
f"{dependency_check_report} for more details."
245242
)
246243

‎test/dlc_tests/sanity/test_safety_check.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -125,7 +125,7 @@ def _get_latest_package_version(package):
125125
@pytest.mark.model("N/A")
126126
@pytest.mark.skipif(not is_dlc_cicd_context(), reason="Skipping test because it is not running in dlc cicd infra")
127127
@pytest.mark.skipif(not is_mainline_context(),
128-
reason="Skipping the test to decrease the number of calls to the Safety Check DB. "
128+
reason="Skipping the test to decrease the number of calls to the Safety Check DB. "
129129
"Test will be executed in the 'mainline' pipeline only")
130130
def test_safety(image):
131131
"""

0 commit comments

Comments
 (0)