Skip to content

Commit 97be952

Browse files
saimidusaimidu
authored
update: Update buildspec for TF 1.15.0 (#265)
* Change path for entrypoint script * Change path for deep learning container script * Change build context to folder containing dockerfiles * Update buildspec * Update buildspec-release
1 parent 90a7b84 commit 97be952

7 files changed

+60
-44
lines changed

‎buildspec-release.yml

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ version: 0.2
22

33
env:
44
variables:
5-
FRAMEWORK_VERSION: '1.13.1'
5+
FRAMEWORK_VERSION: '1.15.0'
66
GPU_INSTANCE_TYPE: 'ml.p2.xlarge'
77
SETUP_FILE: 'setup_cmds.sh'
88
SETUP_CMDS: '#!/bin/bash\npip install --upgrade pip\npip install -U -e .\npip install -U -e .[test]'
@@ -60,21 +60,21 @@ phases:
6060
echo '[{
6161
"repository": "sagemaker-tensorflow-scriptmode",
6262
"tags": [{
63-
"source": "1.13.1-cpu-py2",
64-
"dest": ["1.13.1-cpu-py2", "1.13-cpu-py2", "1.13.1-cpu-py2-'${CODEBUILD_BUILD_ID#*:}'"]
63+
"source": "1.15.0-cpu-py2",
64+
"dest": ["1.15.0-cpu-py2", "1.15-cpu-py2", "1.15.0-cpu-py2-'${CODEBUILD_BUILD_ID#*:}'"]
6565
},{
66-
"source": "1.13.1-cpu-py3",
67-
"dest": ["1.13.1-cpu-py3", "1.13-cpu-py3", "1.13.1-cpu-py3-'${CODEBUILD_BUILD_ID#*:}'"]
66+
"source": "1.15.0-cpu-py3",
67+
"dest": ["1.15.0-cpu-py3", "1.15-cpu-py3", "1.15.0-cpu-py3-'${CODEBUILD_BUILD_ID#*:}'"]
6868
},{
69-
"source": "1.13.1-gpu-py2",
70-
"dest": ["1.13.1-gpu-py2", "1.13-gpu-py2", "1.13.1-gpu-py2-'${CODEBUILD_BUILD_ID#*:}'"]
69+
"source": "1.15.0-gpu-py2",
70+
"dest": ["1.15.0-gpu-py2", "1.15-gpu-py2", "1.15.0-gpu-py2-'${CODEBUILD_BUILD_ID#*:}'"]
7171
},{
72-
"source": "1.13.1-gpu-py3",
73-
"dest": ["1.13.1-gpu-py3", "1.13-gpu-py3", "1.13.1-gpu-py3-'${CODEBUILD_BUILD_ID#*:}'"]
72+
"source": "1.15.0-gpu-py3",
73+
"dest": ["1.15.0-gpu-py3", "1.15-gpu-py3", "1.15.0-gpu-py3-'${CODEBUILD_BUILD_ID#*:}'"]
7474
}],
7575
"test": [
76-
"IGNORE_COVERAGE=- tox -e py36 -- -m deploy_test test/integration/sagemaker -n 4 --region {region} --account-id {aws-id} --instance-type {cpu-instance-type} --docker-base-name sagemaker-tensorflow-scriptmode --framework-version 1.13.1 --processor cpu --py-version 2,3",
77-
"IGNORE_COVERAGE=- tox -e py36 -- -m deploy_test test/integration/sagemaker -n 4 --region {region} --account-id {aws-id} --docker-base-name sagemaker-tensorflow-scriptmode --framework-version 1.13.1 --processor gpu --py-version 2,3"
76+
"IGNORE_COVERAGE=- tox -e py36 -- -m deploy_test test/integration/sagemaker -n 4 --region {region} --account-id {aws-id} --instance-type {cpu-instance-type} --docker-base-name sagemaker-tensorflow-scriptmode --framework-version 1.15.0 --processor cpu --py-version 2,3",
77+
"IGNORE_COVERAGE=- tox -e py36 -- -m deploy_test test/integration/sagemaker -n 4 --region {region} --account-id {aws-id} --docker-base-name sagemaker-tensorflow-scriptmode --framework-version 1.15.0 --processor gpu --py-version 2,3"
7878
]
7979
}]' > deployments.json
8080

‎buildspec.yml

Lines changed: 48 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,7 @@ version: 0.2
22

33
env:
44
variables:
5-
FRAMEWORK_VERSION: '1.13.1'
6-
CPU_FRAMEWORK_BINARY: 'https://s3-us-west-2.amazonaws.com/tensorflow-aws/1.13/AmazonLinux/cpu/latest-patch-latest-patch/tensorflow-1.13.1-cp36-cp36m-linux_x86_64.whl'
7-
CPU_PY_VERSION: '3'
8-
GPU_FRAMEWORK_BINARY: 'https://s3-us-west-2.amazonaws.com/tensorflow-aws/1.13/AmazonLinux/gpu/latest-patch-latest-patch/tensorflow-1.13.1-cp36-cp36m-linux_x86_64.whl'
9-
GPU_PY_VERSION: '3'
5+
FRAMEWORK_VERSION: '1.15.0'
106
ECR_REPO: 'sagemaker-test'
117
GITHUB_REPO: 'sagemaker-tensorflow-container'
128
SETUP_FILE: 'setup_cmds.sh'
@@ -34,42 +30,56 @@ phases:
3430
- tox -e py36,py27 test/unit
3531

3632
# Create pip archive
37-
- build_dir="docker/$FRAMEWORK_VERSION"
33+
- root_dir=$(pwd)
3834
- build_id="$(echo $CODEBUILD_BUILD_ID | sed -e 's/:/-/g')"
3935
- python3 setup.py sdist
4036
- tar_name=$(ls dist)
41-
- cp dist/$tar_name $build_dir
4237

43-
# build cpu image
44-
- cpu_dockerfile="Dockerfile.cpu"
38+
# Find build artifacts
39+
- build_artifacts=$root_dir/docker/artifacts
4540

46-
# Download framework binary
47-
- cpu_fw_binary=$(basename $CPU_FRAMEWORK_BINARY)
48-
- wget -O $build_dir/$cpu_fw_binary $CPU_FRAMEWORK_BINARY
49-
50-
- CPU_TAG="$FRAMEWORK_VERSION-cpu-py$CPU_PY_VERSION-$build_id"
41+
# build py2 images
5142

43+
# prepare build context
44+
- build_dir="$root_dir/docker/$FRAMEWORK_VERSION/py2"
45+
- cp $root_dir/dist/$tar_name $build_dir
46+
- cp $build_artifacts/* $build_dir/
5247
- cd $build_dir
53-
- docker build -f $cpu_dockerfile --build-arg framework_support_installable=$tar_name --build-arg py_version=$CPU_PY_VERSION --build-arg framework_installable=$cpu_fw_binary -t $PREPROD_IMAGE:$CPU_TAG .
54-
- cd ../../
48+
49+
# build cpu image
50+
- cpu_dockerfile="Dockerfile.cpu"
51+
- CPU_TAG_PY2="$FRAMEWORK_VERSION-cpu-py2-$build_id"
52+
- docker build -f $cpu_dockerfile -t $PREPROD_IMAGE:$CPU_TAG_PY2 .
5553

5654
# build gpu image
5755
- gpu_dockerfile="Dockerfile.gpu"
56+
- GPU_TAG_PY2="$FRAMEWORK_VERSION-gpu-py2-$build_id"
57+
- docker build -f $gpu_dockerfile -t $PREPROD_IMAGE:$GPU_TAG_PY2 .
5858

59-
# Download framework binary
60-
- gpu_fw_binary=$(basename $GPU_FRAMEWORK_BINARY)
61-
- wget -O $build_dir/$gpu_fw_binary $GPU_FRAMEWORK_BINARY
62-
63-
- GPU_TAG="$FRAMEWORK_VERSION-gpu-py$GPU_PY_VERSION-$build_id"
59+
# build py3 images
6460

61+
# prepare build context
62+
- build_dir="$root_dir/docker/$FRAMEWORK_VERSION/py3"
63+
- cp $root_dir/dist/$tar_name $build_dir
64+
- cp $build_artifacts/* $build_dir/
6565
- cd $build_dir
66-
- docker build -f $gpu_dockerfile --build-arg framework_support_installable=$tar_name --build-arg py_version=$GPU_PY_VERSION --build-arg framework_installable=$gpu_fw_binary -t $PREPROD_IMAGE:$GPU_TAG .
67-
- cd ../../
66+
67+
# build cpu image
68+
- cpu_dockerfile="Dockerfile.cpu"
69+
- CPU_TAG_PY3="$FRAMEWORK_VERSION-cpu-py3-$build_id"
70+
- docker build -f $cpu_dockerfile -t $PREPROD_IMAGE:$CPU_TAG_PY3 .
71+
72+
# build gpu image
73+
- gpu_dockerfile="Dockerfile.gpu"
74+
- GPU_TAG_PY3="$FRAMEWORK_VERSION-gpu-py3-$build_id"
75+
- docker build -f $gpu_dockerfile -t $PREPROD_IMAGE:$GPU_TAG_PY3 .
6876

6977
# push images to ecr
7078
- $(aws ecr get-login --registry-ids $ACCOUNT --no-include-email --region $AWS_DEFAULT_REGION)
71-
- docker push $PREPROD_IMAGE:$CPU_TAG
72-
- docker push $PREPROD_IMAGE:$GPU_TAG
79+
- docker push $PREPROD_IMAGE:$CPU_TAG_PY2
80+
- docker push $PREPROD_IMAGE:$GPU_TAG_PY2
81+
- docker push $PREPROD_IMAGE:$CPU_TAG_PY3
82+
- docker push $PREPROD_IMAGE:$GPU_TAG_PY3
7383

7484
# launch remote gpu instance
7585
- instance_type='p2.xlarge'
@@ -79,7 +89,8 @@ phases:
7989
# run cpu integration tests
8090
- |
8191
if has-matching-changes "test/" "tests/" "src/*.py" "docker/*" "buildspec.yml"; then
82-
pytest test/integration/local --region $AWS_DEFAULT_REGION --docker-base-name $PREPROD_IMAGE --tag $CPU_TAG --framework-version $FRAMEWORK_VERSION --py-version $CPU_PY_VERSION --processor cpu
92+
pytest test/integration/local --region $AWS_DEFAULT_REGION --docker-base-name $PREPROD_IMAGE --tag $CPU_TAG_PY2 --framework-version $FRAMEWORK_VERSION --py-version 2 --processor cpu
93+
pytest test/integration/local --region $AWS_DEFAULT_REGION --docker-base-name $PREPROD_IMAGE --tag $CPU_TAG_PY3 --framework-version $FRAMEWORK_VERSION --py-version 3 --processor cpu
8394
else
8495
echo "skipping cpu integration tests"
8596
fi
@@ -88,7 +99,9 @@ phases:
8899
- |
89100
if has-matching-changes "test/" "tests/" "src/*.py" "docker/*" "buildspec.yml"; then
90101
printf "$SETUP_CMDS" > $SETUP_FILE
91-
cmd="pytest test/integration/local --region $AWS_DEFAULT_REGION --docker-base-name $PREPROD_IMAGE --tag $GPU_TAG --framework-version $FRAMEWORK_VERSION --py-version $GPU_PY_VERSION --processor gpu"
102+
cmd="pytest test/integration/local --region $AWS_DEFAULT_REGION --docker-base-name $PREPROD_IMAGE --tag $GPU_TAG_PY2 --framework-version $FRAMEWORK_VERSION --py-version 2 --processor gpu"
103+
remote-test --github-repo $GITHUB_REPO --test-cmd "$cmd" --setup-file $SETUP_FILE --pr-number "$PR_NUM"
104+
cmd="pytest test/integration/local --region $AWS_DEFAULT_REGION --docker-base-name $PREPROD_IMAGE --tag $GPU_TAG_PY3 --framework-version $FRAMEWORK_VERSION --py-version 3 --processor gpu"
92105
remote-test --github-repo $GITHUB_REPO --test-cmd "$cmd" --setup-file $SETUP_FILE --pr-number "$PR_NUM"
93106
else
94107
echo "skipping gpu integration tests"
@@ -97,8 +110,10 @@ phases:
97110
# run sagemaker tests
98111
- |
99112
if has-matching-changes "test/" "tests/" "src/*.py" "docker/*" "buildspec.yml"; then
100-
pytest test/integration/sagemaker -n 8 --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --account-id $ACCOUNT --tag $CPU_TAG --py-version $CPU_PY_VERSION --processor cpu
101-
pytest test/integration/sagemaker -n 8 --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --account-id $ACCOUNT --tag $GPU_TAG --py-version $GPU_PY_VERSION --processor gpu
113+
pytest test/integration/sagemaker -n 8 --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --account-id $ACCOUNT --tag $CPU_TAG_PY2 --py-version 2 --processor cpu
114+
pytest test/integration/sagemaker -n 8 --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --account-id $ACCOUNT --tag $GPU_TAG_PY2 --py-version 2 --processor gpu
115+
pytest test/integration/sagemaker -n 8 --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --account-id $ACCOUNT --tag $CPU_TAG_PY3 --py-version 3 --processor cpu
116+
pytest test/integration/sagemaker -n 8 --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --account-id $ACCOUNT --tag $GPU_TAG_PY3 --py-version 3 --processor gpu
102117
else
103118
echo "skipping sagemaker tests"
104119
fi
@@ -109,5 +124,7 @@ phases:
109124
- cleanup-key-pairs
110125

111126
# remove ecr image
112-
- aws ecr batch-delete-image --repository-name $ECR_REPO --region $AWS_DEFAULT_REGION --image-ids imageTag=$CPU_TAG
113-
- aws ecr batch-delete-image --repository-name $ECR_REPO --region $AWS_DEFAULT_REGION --image-ids imageTag=$GPU_TAG
127+
- aws ecr batch-delete-image --repository-name $ECR_REPO --region $AWS_DEFAULT_REGION --image-ids imageTag=$CPU_TAG_PY2
128+
- aws ecr batch-delete-image --repository-name $ECR_REPO --region $AWS_DEFAULT_REGION --image-ids imageTag=$GPU_TAG_PY2
129+
- aws ecr batch-delete-image --repository-name $ECR_REPO --region $AWS_DEFAULT_REGION --image-ids imageTag=$CPU_TAG_PY3
130+
- aws ecr batch-delete-image --repository-name $ECR_REPO --region $AWS_DEFAULT_REGION --image-ids imageTag=$GPU_TAG_PY3

‎docker/__init__.py

Whitespace-only changes.

‎docker/build_artifacts/__init__.py

Whitespace-only changes.

‎src/sagemaker_tensorflow_container/deep_learning_container.py renamed to ‎docker/build_artifacts/deep_learning_container.py

File renamed without changes.

‎docker/1.15.0/py2/dockerd-entrypoint.py renamed to ‎docker/build_artifacts/dockerd-entrypoint.py

File renamed without changes.

‎test/unit/test_deep_learning_containers.py renamed to ‎test/unit/test_deep_learning_container.py

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,11 +14,10 @@
1414

1515
import unittest
1616

17+
from docker.build_artifacts import deep_learning_container as deep_learning_container_to_test
1718
import pytest
1819
import requests
1920

20-
from sagemaker_tensorflow_container import deep_learning_container as deep_learning_container_to_test
21-
2221

2322
@pytest.fixture(name='fixture_valid_instance_id')
2423
def fixture_valid_instance_id(requests_mock):

0 commit comments

Comments
 (0)