From b9bec16d912e95a9088cd1c8149d57154828d405 Mon Sep 17 00:00:00 2001 From: Sai Parthasarathy Miduthuri Date: Mon, 30 Dec 2019 12:16:31 -0800 Subject: [PATCH 1/8] Change path for entrypoint script --- docker/{1.15.0/py2 => }/dockerd-entrypoint.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename docker/{1.15.0/py2 => }/dockerd-entrypoint.py (100%) diff --git a/docker/1.15.0/py2/dockerd-entrypoint.py b/docker/dockerd-entrypoint.py similarity index 100% rename from docker/1.15.0/py2/dockerd-entrypoint.py rename to docker/dockerd-entrypoint.py From 98dedefb48867978742d9f752cf6b8d4b4404eb6 Mon Sep 17 00:00:00 2001 From: Sai Parthasarathy Miduthuri Date: Mon, 30 Dec 2019 12:24:12 -0800 Subject: [PATCH 2/8] Change path for deep learning container script --- .../deep_learning_container.py | 0 test/unit/test_deep_learning_containers.py | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename {src/sagemaker_tensorflow_container => docker}/deep_learning_container.py (100%) diff --git a/src/sagemaker_tensorflow_container/deep_learning_container.py b/docker/deep_learning_container.py similarity index 100% rename from src/sagemaker_tensorflow_container/deep_learning_container.py rename to docker/deep_learning_container.py diff --git a/test/unit/test_deep_learning_containers.py b/test/unit/test_deep_learning_containers.py index 8d6fe08e..3c01dd72 100644 --- a/test/unit/test_deep_learning_containers.py +++ b/test/unit/test_deep_learning_containers.py @@ -17,7 +17,7 @@ import pytest import requests -from sagemaker_tensorflow_container import deep_learning_container as deep_learning_container_to_test +from docker import deep_learning_container as deep_learning_container_to_test @pytest.fixture(name='fixture_valid_instance_id') From 8b760b3855f4ed1b196d16aedea7b80bf6036c9a Mon Sep 17 00:00:00 2001 From: Sai Parthasarathy Miduthuri Date: Thu, 2 Jan 2020 17:32:56 -0800 Subject: [PATCH 3/8] Change build context to folder containing dockerfiles --- docker/{ => build_artifacts}/deep_learning_container.py | 0 docker/{ => build_artifacts}/dockerd-entrypoint.py | 0 test/unit/test_deep_learning_containers.py | 2 +- 3 files changed, 1 insertion(+), 1 deletion(-) rename docker/{ => build_artifacts}/deep_learning_container.py (100%) rename docker/{ => build_artifacts}/dockerd-entrypoint.py (100%) diff --git a/docker/deep_learning_container.py b/docker/build_artifacts/deep_learning_container.py similarity index 100% rename from docker/deep_learning_container.py rename to docker/build_artifacts/deep_learning_container.py diff --git a/docker/dockerd-entrypoint.py b/docker/build_artifacts/dockerd-entrypoint.py similarity index 100% rename from docker/dockerd-entrypoint.py rename to docker/build_artifacts/dockerd-entrypoint.py diff --git a/test/unit/test_deep_learning_containers.py b/test/unit/test_deep_learning_containers.py index 3c01dd72..821218c6 100644 --- a/test/unit/test_deep_learning_containers.py +++ b/test/unit/test_deep_learning_containers.py @@ -17,7 +17,7 @@ import pytest import requests -from docker import deep_learning_container as deep_learning_container_to_test +from .docker.build_artifacts import deep_learning_container as deep_learning_container_to_test @pytest.fixture(name='fixture_valid_instance_id') From da1a091b0d0a83eedc50ec37f14f4ece743533af Mon Sep 17 00:00:00 2001 From: Sai Parthasarathy Miduthuri Date: Fri, 3 Jan 2020 14:44:36 -0800 Subject: [PATCH 4/8] Update buildspec --- buildspec.yml | 81 +++++++++++++++++++++++++++++++-------------------- 1 file changed, 49 insertions(+), 32 deletions(-) diff --git a/buildspec.yml b/buildspec.yml index 214cdcca..c69feb5f 100644 --- a/buildspec.yml +++ b/buildspec.yml @@ -1,12 +1,8 @@ -version: 0.2 +version: 0.3 env: variables: - FRAMEWORK_VERSION: '1.13.1' - CPU_FRAMEWORK_BINARY: 'https://s3-us-west-2.amazonaws.com/tensorflow-aws/1.13/AmazonLinux/cpu/latest-patch-latest-patch/tensorflow-1.13.1-cp36-cp36m-linux_x86_64.whl' - CPU_PY_VERSION: '3' - GPU_FRAMEWORK_BINARY: 'https://s3-us-west-2.amazonaws.com/tensorflow-aws/1.13/AmazonLinux/gpu/latest-patch-latest-patch/tensorflow-1.13.1-cp36-cp36m-linux_x86_64.whl' - GPU_PY_VERSION: '3' + FRAMEWORK_VERSION: '1.15.0' ECR_REPO: 'sagemaker-test' GITHUB_REPO: 'sagemaker-tensorflow-container' SETUP_FILE: 'setup_cmds.sh' @@ -34,42 +30,56 @@ phases: - tox -e py36,py27 test/unit # Create pip archive - - build_dir="docker/$FRAMEWORK_VERSION" + - root_dir=$(pwd) - build_id="$(echo $CODEBUILD_BUILD_ID | sed -e 's/:/-/g')" - python3 setup.py sdist - tar_name=$(ls dist) - - cp dist/$tar_name $build_dir - # build cpu image - - cpu_dockerfile="Dockerfile.cpu" + # Find build artifacts + - build_artifacts=$root_dir/docker/artifacts - # Download framework binary - - cpu_fw_binary=$(basename $CPU_FRAMEWORK_BINARY) - - wget -O $build_dir/$cpu_fw_binary $CPU_FRAMEWORK_BINARY - - - CPU_TAG="$FRAMEWORK_VERSION-cpu-py$CPU_PY_VERSION-$build_id" + # build py2 images + # prepare build context + - build_dir="$root_dir/docker/$FRAMEWORK_VERSION/py2" + - cp $root_dir/dist/$tar_name $build_dir + - cp $build_artifacts/* $build_dir/ - cd $build_dir - - docker build -f $cpu_dockerfile --build-arg framework_support_installable=$tar_name --build-arg py_version=$CPU_PY_VERSION --build-arg framework_installable=$cpu_fw_binary -t $PREPROD_IMAGE:$CPU_TAG . - - cd ../../ + + # build cpu image + - cpu_dockerfile="Dockerfile.cpu" + - CPU_TAG_PY2="$FRAMEWORK_VERSION-cpu-py2-$build_id" + - docker build -f $cpu_dockerfile -t $PREPROD_IMAGE:$CPU_TAG_PY2 . # build gpu image - gpu_dockerfile="Dockerfile.gpu" + - GPU_TAG_PY2="$FRAMEWORK_VERSION-gpu-py2-$build_id" + - docker build -f $gpu_dockerfile -t $PREPROD_IMAGE:$GPU_TAG_PY2 . - # Download framework binary - - gpu_fw_binary=$(basename $GPU_FRAMEWORK_BINARY) - - wget -O $build_dir/$gpu_fw_binary $GPU_FRAMEWORK_BINARY - - - GPU_TAG="$FRAMEWORK_VERSION-gpu-py$GPU_PY_VERSION-$build_id" + # build py3 images + # prepare build context + - build_dir="$root_dir/docker/$FRAMEWORK_VERSION/py3" + - cp $root_dir/dist/$tar_name $build_dir + - cp $build_artifacts/* $build_dir/ - cd $build_dir - - docker build -f $gpu_dockerfile --build-arg framework_support_installable=$tar_name --build-arg py_version=$GPU_PY_VERSION --build-arg framework_installable=$gpu_fw_binary -t $PREPROD_IMAGE:$GPU_TAG . - - cd ../../ + + # build cpu image + - cpu_dockerfile="Dockerfile.cpu" + - CPU_TAG_PY3="$FRAMEWORK_VERSION-cpu-py3-$build_id" + - docker build -f $cpu_dockerfile -t $PREPROD_IMAGE:$CPU_TAG_PY3 . + + # build gpu image + - gpu_dockerfile="Dockerfile.gpu" + - GPU_TAG_PY3="$FRAMEWORK_VERSION-gpu-py3-$build_id" + - docker build -f $gpu_dockerfile -t $PREPROD_IMAGE:$GPU_TAG_PY3 . # push images to ecr - $(aws ecr get-login --registry-ids $ACCOUNT --no-include-email --region $AWS_DEFAULT_REGION) - - docker push $PREPROD_IMAGE:$CPU_TAG - - docker push $PREPROD_IMAGE:$GPU_TAG + - docker push $PREPROD_IMAGE:$CPU_TAG_PY2 + - docker push $PREPROD_IMAGE:$GPU_TAG_PY2 + - docker push $PREPROD_IMAGE:$CPU_TAG_PY3 + - docker push $PREPROD_IMAGE:$GPU_TAG_PY3 # launch remote gpu instance - instance_type='p2.xlarge' @@ -79,7 +89,8 @@ phases: # run cpu integration tests - | if has-matching-changes "test/" "tests/" "src/*.py" "docker/*" "buildspec.yml"; then - pytest test/integration/local --region $AWS_DEFAULT_REGION --docker-base-name $PREPROD_IMAGE --tag $CPU_TAG --framework-version $FRAMEWORK_VERSION --py-version $CPU_PY_VERSION --processor cpu + pytest test/integration/local --region $AWS_DEFAULT_REGION --docker-base-name $PREPROD_IMAGE --tag $CPU_TAG_PY2 --framework-version $FRAMEWORK_VERSION --py-version 2 --processor cpu + pytest test/integration/local --region $AWS_DEFAULT_REGION --docker-base-name $PREPROD_IMAGE --tag $CPU_TAG_PY3 --framework-version $FRAMEWORK_VERSION --py-version 3 --processor cpu else echo "skipping cpu integration tests" fi @@ -88,7 +99,9 @@ phases: - | if has-matching-changes "test/" "tests/" "src/*.py" "docker/*" "buildspec.yml"; then printf "$SETUP_CMDS" > $SETUP_FILE - cmd="pytest test/integration/local --region $AWS_DEFAULT_REGION --docker-base-name $PREPROD_IMAGE --tag $GPU_TAG --framework-version $FRAMEWORK_VERSION --py-version $GPU_PY_VERSION --processor gpu" + cmd="pytest test/integration/local --region $AWS_DEFAULT_REGION --docker-base-name $PREPROD_IMAGE --tag $GPU_TAG_PY2 --framework-version $FRAMEWORK_VERSION --py-version 2 --processor gpu" + remote-test --github-repo $GITHUB_REPO --test-cmd "$cmd" --setup-file $SETUP_FILE --pr-number "$PR_NUM" + cmd="pytest test/integration/local --region $AWS_DEFAULT_REGION --docker-base-name $PREPROD_IMAGE --tag $GPU_TAG_PY3 --framework-version $FRAMEWORK_VERSION --py-version 3 --processor gpu" remote-test --github-repo $GITHUB_REPO --test-cmd "$cmd" --setup-file $SETUP_FILE --pr-number "$PR_NUM" else echo "skipping gpu integration tests" @@ -97,8 +110,10 @@ phases: # run sagemaker tests - | if has-matching-changes "test/" "tests/" "src/*.py" "docker/*" "buildspec.yml"; then - pytest test/integration/sagemaker -n 8 --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --account-id $ACCOUNT --tag $CPU_TAG --py-version $CPU_PY_VERSION --processor cpu - pytest test/integration/sagemaker -n 8 --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --account-id $ACCOUNT --tag $GPU_TAG --py-version $GPU_PY_VERSION --processor gpu + pytest test/integration/sagemaker -n 8 --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --account-id $ACCOUNT --tag $CPU_TAG_PY2 --py-version 2 --processor cpu + pytest test/integration/sagemaker -n 8 --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --account-id $ACCOUNT --tag $GPU_TAG_PY2 --py-version 2 --processor gpu + pytest test/integration/sagemaker -n 8 --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --account-id $ACCOUNT --tag $CPU_TAG_PY3 --py-version 3 --processor cpu + pytest test/integration/sagemaker -n 8 --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --account-id $ACCOUNT --tag $GPU_TAG_PY3 --py-version 3 --processor gpu else echo "skipping sagemaker tests" fi @@ -109,5 +124,7 @@ phases: - cleanup-key-pairs # remove ecr image - - aws ecr batch-delete-image --repository-name $ECR_REPO --region $AWS_DEFAULT_REGION --image-ids imageTag=$CPU_TAG - - aws ecr batch-delete-image --repository-name $ECR_REPO --region $AWS_DEFAULT_REGION --image-ids imageTag=$GPU_TAG + - aws ecr batch-delete-image --repository-name $ECR_REPO --region $AWS_DEFAULT_REGION --image-ids imageTag=$CPU_TAG_PY2 + - aws ecr batch-delete-image --repository-name $ECR_REPO --region $AWS_DEFAULT_REGION --image-ids imageTag=$GPU_TAG_PY2 + - aws ecr batch-delete-image --repository-name $ECR_REPO --region $AWS_DEFAULT_REGION --image-ids imageTag=$CPU_TAG_PY3 + - aws ecr batch-delete-image --repository-name $ECR_REPO --region $AWS_DEFAULT_REGION --image-ids imageTag=$GPU_TAG_PY3 From 52951afd79a60d93691a6548456b8740623561e1 Mon Sep 17 00:00:00 2001 From: Sai Parthasarathy Miduthuri Date: Fri, 3 Jan 2020 17:37:33 -0800 Subject: [PATCH 5/8] Correct import for dlc test --- test/unit/test_deep_learning_containers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/unit/test_deep_learning_containers.py b/test/unit/test_deep_learning_containers.py index 821218c6..7d85c9d6 100644 --- a/test/unit/test_deep_learning_containers.py +++ b/test/unit/test_deep_learning_containers.py @@ -17,7 +17,7 @@ import pytest import requests -from .docker.build_artifacts import deep_learning_container as deep_learning_container_to_test +from docker.build_artifacts import deep_learning_container as deep_learning_container_to_test @pytest.fixture(name='fixture_valid_instance_id') From bee22bfaeb082f944cf472b49a36c7841e55d7d6 Mon Sep 17 00:00:00 2001 From: Sai Parthasarathy Miduthuri Date: Mon, 6 Jan 2020 17:38:33 -0800 Subject: [PATCH 6/8] Add init and comply with flake8 --- docker/__init__.py | 0 ..._learning_containers.py => test_deep_learning_container.py} | 3 +-- 2 files changed, 1 insertion(+), 2 deletions(-) create mode 100644 docker/__init__.py rename test/unit/{test_deep_learning_containers.py => test_deep_learning_container.py} (99%) diff --git a/docker/__init__.py b/docker/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/test/unit/test_deep_learning_containers.py b/test/unit/test_deep_learning_container.py similarity index 99% rename from test/unit/test_deep_learning_containers.py rename to test/unit/test_deep_learning_container.py index 7d85c9d6..7d5d7d86 100644 --- a/test/unit/test_deep_learning_containers.py +++ b/test/unit/test_deep_learning_container.py @@ -14,11 +14,10 @@ import unittest +from docker.build_artifacts import deep_learning_container as deep_learning_container_to_test import pytest import requests -from docker.build_artifacts import deep_learning_container as deep_learning_container_to_test - @pytest.fixture(name='fixture_valid_instance_id') def fixture_valid_instance_id(requests_mock): From dd640d8513604c8212a71fe3c588af109ce92165 Mon Sep 17 00:00:00 2001 From: Sai Parthasarathy Miduthuri Date: Mon, 6 Jan 2020 17:46:48 -0800 Subject: [PATCH 7/8] Add init inside build artifacts dir --- docker/build_artifacts/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 docker/build_artifacts/__init__.py diff --git a/docker/build_artifacts/__init__.py b/docker/build_artifacts/__init__.py new file mode 100644 index 00000000..e69de29b From b7bd98469936a6ab9d621dcb90a457616f6b0790 Mon Sep 17 00:00:00 2001 From: Sai Parthasarathy Miduthuri Date: Tue, 7 Jan 2020 09:21:50 -0800 Subject: [PATCH 8/8] Update buildspec-release --- buildspec-release.yml | 22 +++++++++++----------- buildspec.yml | 2 +- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/buildspec-release.yml b/buildspec-release.yml index a4ff55a5..2e5a9a86 100644 --- a/buildspec-release.yml +++ b/buildspec-release.yml @@ -2,7 +2,7 @@ version: 0.2 env: variables: - FRAMEWORK_VERSION: '1.13.1' + FRAMEWORK_VERSION: '1.15.0' GPU_INSTANCE_TYPE: 'ml.p2.xlarge' SETUP_FILE: 'setup_cmds.sh' SETUP_CMDS: '#!/bin/bash\npip install --upgrade pip\npip install -U -e .\npip install -U -e .[test]' @@ -60,21 +60,21 @@ phases: echo '[{ "repository": "sagemaker-tensorflow-scriptmode", "tags": [{ - "source": "1.13.1-cpu-py2", - "dest": ["1.13.1-cpu-py2", "1.13-cpu-py2", "1.13.1-cpu-py2-'${CODEBUILD_BUILD_ID#*:}'"] + "source": "1.15.0-cpu-py2", + "dest": ["1.15.0-cpu-py2", "1.15-cpu-py2", "1.15.0-cpu-py2-'${CODEBUILD_BUILD_ID#*:}'"] },{ - "source": "1.13.1-cpu-py3", - "dest": ["1.13.1-cpu-py3", "1.13-cpu-py3", "1.13.1-cpu-py3-'${CODEBUILD_BUILD_ID#*:}'"] + "source": "1.15.0-cpu-py3", + "dest": ["1.15.0-cpu-py3", "1.15-cpu-py3", "1.15.0-cpu-py3-'${CODEBUILD_BUILD_ID#*:}'"] },{ - "source": "1.13.1-gpu-py2", - "dest": ["1.13.1-gpu-py2", "1.13-gpu-py2", "1.13.1-gpu-py2-'${CODEBUILD_BUILD_ID#*:}'"] + "source": "1.15.0-gpu-py2", + "dest": ["1.15.0-gpu-py2", "1.15-gpu-py2", "1.15.0-gpu-py2-'${CODEBUILD_BUILD_ID#*:}'"] },{ - "source": "1.13.1-gpu-py3", - "dest": ["1.13.1-gpu-py3", "1.13-gpu-py3", "1.13.1-gpu-py3-'${CODEBUILD_BUILD_ID#*:}'"] + "source": "1.15.0-gpu-py3", + "dest": ["1.15.0-gpu-py3", "1.15-gpu-py3", "1.15.0-gpu-py3-'${CODEBUILD_BUILD_ID#*:}'"] }], "test": [ - "IGNORE_COVERAGE=- tox -e py36 -- -m deploy_test test/integration/sagemaker -n 4 --region {region} --account-id {aws-id} --instance-type {cpu-instance-type} --docker-base-name sagemaker-tensorflow-scriptmode --framework-version 1.13.1 --processor cpu --py-version 2,3", - "IGNORE_COVERAGE=- tox -e py36 -- -m deploy_test test/integration/sagemaker -n 4 --region {region} --account-id {aws-id} --docker-base-name sagemaker-tensorflow-scriptmode --framework-version 1.13.1 --processor gpu --py-version 2,3" + "IGNORE_COVERAGE=- tox -e py36 -- -m deploy_test test/integration/sagemaker -n 4 --region {region} --account-id {aws-id} --instance-type {cpu-instance-type} --docker-base-name sagemaker-tensorflow-scriptmode --framework-version 1.15.0 --processor cpu --py-version 2,3", + "IGNORE_COVERAGE=- tox -e py36 -- -m deploy_test test/integration/sagemaker -n 4 --region {region} --account-id {aws-id} --docker-base-name sagemaker-tensorflow-scriptmode --framework-version 1.15.0 --processor gpu --py-version 2,3" ] }]' > deployments.json diff --git a/buildspec.yml b/buildspec.yml index c69feb5f..d59393c3 100644 --- a/buildspec.yml +++ b/buildspec.yml @@ -1,4 +1,4 @@ -version: 0.3 +version: 0.2 env: variables: