From 89dd978e21f057763dc7d5238b36bc5e1c0de02c Mon Sep 17 00:00:00 2001 From: Denis Davydenko Date: Fri, 10 Jan 2020 15:22:10 -0800 Subject: [PATCH] Revert "Revert "update: Update buildspec for TF 1.15.0 (#265)"" --- buildspec-release.yml | 22 +++--- buildspec.yml | 79 +++++++++++-------- docker/__init__.py | 0 docker/build_artifacts/__init__.py | 0 .../deep_learning_container.py | 0 .../dockerd-entrypoint.py | 0 ...ers.py => test_deep_learning_container.py} | 3 +- 7 files changed, 60 insertions(+), 44 deletions(-) create mode 100644 docker/__init__.py create mode 100644 docker/build_artifacts/__init__.py rename {src/sagemaker_tensorflow_container => docker/build_artifacts}/deep_learning_container.py (100%) rename docker/{1.15.0/py2 => build_artifacts}/dockerd-entrypoint.py (100%) rename test/unit/{test_deep_learning_containers.py => test_deep_learning_container.py} (98%) diff --git a/buildspec-release.yml b/buildspec-release.yml index a4ff55a5..2e5a9a86 100644 --- a/buildspec-release.yml +++ b/buildspec-release.yml @@ -2,7 +2,7 @@ version: 0.2 env: variables: - FRAMEWORK_VERSION: '1.13.1' + FRAMEWORK_VERSION: '1.15.0' GPU_INSTANCE_TYPE: 'ml.p2.xlarge' SETUP_FILE: 'setup_cmds.sh' SETUP_CMDS: '#!/bin/bash\npip install --upgrade pip\npip install -U -e .\npip install -U -e .[test]' @@ -60,21 +60,21 @@ phases: echo '[{ "repository": "sagemaker-tensorflow-scriptmode", "tags": [{ - "source": "1.13.1-cpu-py2", - "dest": ["1.13.1-cpu-py2", "1.13-cpu-py2", "1.13.1-cpu-py2-'${CODEBUILD_BUILD_ID#*:}'"] + "source": "1.15.0-cpu-py2", + "dest": ["1.15.0-cpu-py2", "1.15-cpu-py2", "1.15.0-cpu-py2-'${CODEBUILD_BUILD_ID#*:}'"] },{ - "source": "1.13.1-cpu-py3", - "dest": ["1.13.1-cpu-py3", "1.13-cpu-py3", "1.13.1-cpu-py3-'${CODEBUILD_BUILD_ID#*:}'"] + "source": "1.15.0-cpu-py3", + "dest": ["1.15.0-cpu-py3", "1.15-cpu-py3", "1.15.0-cpu-py3-'${CODEBUILD_BUILD_ID#*:}'"] },{ - "source": "1.13.1-gpu-py2", - "dest": ["1.13.1-gpu-py2", "1.13-gpu-py2", "1.13.1-gpu-py2-'${CODEBUILD_BUILD_ID#*:}'"] + "source": "1.15.0-gpu-py2", + "dest": ["1.15.0-gpu-py2", "1.15-gpu-py2", "1.15.0-gpu-py2-'${CODEBUILD_BUILD_ID#*:}'"] },{ - "source": "1.13.1-gpu-py3", - "dest": ["1.13.1-gpu-py3", "1.13-gpu-py3", "1.13.1-gpu-py3-'${CODEBUILD_BUILD_ID#*:}'"] + "source": "1.15.0-gpu-py3", + "dest": ["1.15.0-gpu-py3", "1.15-gpu-py3", "1.15.0-gpu-py3-'${CODEBUILD_BUILD_ID#*:}'"] }], "test": [ - "IGNORE_COVERAGE=- tox -e py36 -- -m deploy_test test/integration/sagemaker -n 4 --region {region} --account-id {aws-id} --instance-type {cpu-instance-type} --docker-base-name sagemaker-tensorflow-scriptmode --framework-version 1.13.1 --processor cpu --py-version 2,3", - "IGNORE_COVERAGE=- tox -e py36 -- -m deploy_test test/integration/sagemaker -n 4 --region {region} --account-id {aws-id} --docker-base-name sagemaker-tensorflow-scriptmode --framework-version 1.13.1 --processor gpu --py-version 2,3" + "IGNORE_COVERAGE=- tox -e py36 -- -m deploy_test test/integration/sagemaker -n 4 --region {region} --account-id {aws-id} --instance-type {cpu-instance-type} --docker-base-name sagemaker-tensorflow-scriptmode --framework-version 1.15.0 --processor cpu --py-version 2,3", + "IGNORE_COVERAGE=- tox -e py36 -- -m deploy_test test/integration/sagemaker -n 4 --region {region} --account-id {aws-id} --docker-base-name sagemaker-tensorflow-scriptmode --framework-version 1.15.0 --processor gpu --py-version 2,3" ] }]' > deployments.json diff --git a/buildspec.yml b/buildspec.yml index 214cdcca..d59393c3 100644 --- a/buildspec.yml +++ b/buildspec.yml @@ -2,11 +2,7 @@ version: 0.2 env: variables: - FRAMEWORK_VERSION: '1.13.1' - CPU_FRAMEWORK_BINARY: 'https://s3-us-west-2.amazonaws.com/tensorflow-aws/1.13/AmazonLinux/cpu/latest-patch-latest-patch/tensorflow-1.13.1-cp36-cp36m-linux_x86_64.whl' - CPU_PY_VERSION: '3' - GPU_FRAMEWORK_BINARY: 'https://s3-us-west-2.amazonaws.com/tensorflow-aws/1.13/AmazonLinux/gpu/latest-patch-latest-patch/tensorflow-1.13.1-cp36-cp36m-linux_x86_64.whl' - GPU_PY_VERSION: '3' + FRAMEWORK_VERSION: '1.15.0' ECR_REPO: 'sagemaker-test' GITHUB_REPO: 'sagemaker-tensorflow-container' SETUP_FILE: 'setup_cmds.sh' @@ -34,42 +30,56 @@ phases: - tox -e py36,py27 test/unit # Create pip archive - - build_dir="docker/$FRAMEWORK_VERSION" + - root_dir=$(pwd) - build_id="$(echo $CODEBUILD_BUILD_ID | sed -e 's/:/-/g')" - python3 setup.py sdist - tar_name=$(ls dist) - - cp dist/$tar_name $build_dir - # build cpu image - - cpu_dockerfile="Dockerfile.cpu" + # Find build artifacts + - build_artifacts=$root_dir/docker/artifacts - # Download framework binary - - cpu_fw_binary=$(basename $CPU_FRAMEWORK_BINARY) - - wget -O $build_dir/$cpu_fw_binary $CPU_FRAMEWORK_BINARY - - - CPU_TAG="$FRAMEWORK_VERSION-cpu-py$CPU_PY_VERSION-$build_id" + # build py2 images + # prepare build context + - build_dir="$root_dir/docker/$FRAMEWORK_VERSION/py2" + - cp $root_dir/dist/$tar_name $build_dir + - cp $build_artifacts/* $build_dir/ - cd $build_dir - - docker build -f $cpu_dockerfile --build-arg framework_support_installable=$tar_name --build-arg py_version=$CPU_PY_VERSION --build-arg framework_installable=$cpu_fw_binary -t $PREPROD_IMAGE:$CPU_TAG . - - cd ../../ + + # build cpu image + - cpu_dockerfile="Dockerfile.cpu" + - CPU_TAG_PY2="$FRAMEWORK_VERSION-cpu-py2-$build_id" + - docker build -f $cpu_dockerfile -t $PREPROD_IMAGE:$CPU_TAG_PY2 . # build gpu image - gpu_dockerfile="Dockerfile.gpu" + - GPU_TAG_PY2="$FRAMEWORK_VERSION-gpu-py2-$build_id" + - docker build -f $gpu_dockerfile -t $PREPROD_IMAGE:$GPU_TAG_PY2 . - # Download framework binary - - gpu_fw_binary=$(basename $GPU_FRAMEWORK_BINARY) - - wget -O $build_dir/$gpu_fw_binary $GPU_FRAMEWORK_BINARY - - - GPU_TAG="$FRAMEWORK_VERSION-gpu-py$GPU_PY_VERSION-$build_id" + # build py3 images + # prepare build context + - build_dir="$root_dir/docker/$FRAMEWORK_VERSION/py3" + - cp $root_dir/dist/$tar_name $build_dir + - cp $build_artifacts/* $build_dir/ - cd $build_dir - - docker build -f $gpu_dockerfile --build-arg framework_support_installable=$tar_name --build-arg py_version=$GPU_PY_VERSION --build-arg framework_installable=$gpu_fw_binary -t $PREPROD_IMAGE:$GPU_TAG . - - cd ../../ + + # build cpu image + - cpu_dockerfile="Dockerfile.cpu" + - CPU_TAG_PY3="$FRAMEWORK_VERSION-cpu-py3-$build_id" + - docker build -f $cpu_dockerfile -t $PREPROD_IMAGE:$CPU_TAG_PY3 . + + # build gpu image + - gpu_dockerfile="Dockerfile.gpu" + - GPU_TAG_PY3="$FRAMEWORK_VERSION-gpu-py3-$build_id" + - docker build -f $gpu_dockerfile -t $PREPROD_IMAGE:$GPU_TAG_PY3 . # push images to ecr - $(aws ecr get-login --registry-ids $ACCOUNT --no-include-email --region $AWS_DEFAULT_REGION) - - docker push $PREPROD_IMAGE:$CPU_TAG - - docker push $PREPROD_IMAGE:$GPU_TAG + - docker push $PREPROD_IMAGE:$CPU_TAG_PY2 + - docker push $PREPROD_IMAGE:$GPU_TAG_PY2 + - docker push $PREPROD_IMAGE:$CPU_TAG_PY3 + - docker push $PREPROD_IMAGE:$GPU_TAG_PY3 # launch remote gpu instance - instance_type='p2.xlarge' @@ -79,7 +89,8 @@ phases: # run cpu integration tests - | if has-matching-changes "test/" "tests/" "src/*.py" "docker/*" "buildspec.yml"; then - pytest test/integration/local --region $AWS_DEFAULT_REGION --docker-base-name $PREPROD_IMAGE --tag $CPU_TAG --framework-version $FRAMEWORK_VERSION --py-version $CPU_PY_VERSION --processor cpu + pytest test/integration/local --region $AWS_DEFAULT_REGION --docker-base-name $PREPROD_IMAGE --tag $CPU_TAG_PY2 --framework-version $FRAMEWORK_VERSION --py-version 2 --processor cpu + pytest test/integration/local --region $AWS_DEFAULT_REGION --docker-base-name $PREPROD_IMAGE --tag $CPU_TAG_PY3 --framework-version $FRAMEWORK_VERSION --py-version 3 --processor cpu else echo "skipping cpu integration tests" fi @@ -88,7 +99,9 @@ phases: - | if has-matching-changes "test/" "tests/" "src/*.py" "docker/*" "buildspec.yml"; then printf "$SETUP_CMDS" > $SETUP_FILE - cmd="pytest test/integration/local --region $AWS_DEFAULT_REGION --docker-base-name $PREPROD_IMAGE --tag $GPU_TAG --framework-version $FRAMEWORK_VERSION --py-version $GPU_PY_VERSION --processor gpu" + cmd="pytest test/integration/local --region $AWS_DEFAULT_REGION --docker-base-name $PREPROD_IMAGE --tag $GPU_TAG_PY2 --framework-version $FRAMEWORK_VERSION --py-version 2 --processor gpu" + remote-test --github-repo $GITHUB_REPO --test-cmd "$cmd" --setup-file $SETUP_FILE --pr-number "$PR_NUM" + cmd="pytest test/integration/local --region $AWS_DEFAULT_REGION --docker-base-name $PREPROD_IMAGE --tag $GPU_TAG_PY3 --framework-version $FRAMEWORK_VERSION --py-version 3 --processor gpu" remote-test --github-repo $GITHUB_REPO --test-cmd "$cmd" --setup-file $SETUP_FILE --pr-number "$PR_NUM" else echo "skipping gpu integration tests" @@ -97,8 +110,10 @@ phases: # run sagemaker tests - | if has-matching-changes "test/" "tests/" "src/*.py" "docker/*" "buildspec.yml"; then - pytest test/integration/sagemaker -n 8 --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --account-id $ACCOUNT --tag $CPU_TAG --py-version $CPU_PY_VERSION --processor cpu - pytest test/integration/sagemaker -n 8 --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --account-id $ACCOUNT --tag $GPU_TAG --py-version $GPU_PY_VERSION --processor gpu + pytest test/integration/sagemaker -n 8 --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --account-id $ACCOUNT --tag $CPU_TAG_PY2 --py-version 2 --processor cpu + pytest test/integration/sagemaker -n 8 --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --account-id $ACCOUNT --tag $GPU_TAG_PY2 --py-version 2 --processor gpu + pytest test/integration/sagemaker -n 8 --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --account-id $ACCOUNT --tag $CPU_TAG_PY3 --py-version 3 --processor cpu + pytest test/integration/sagemaker -n 8 --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --account-id $ACCOUNT --tag $GPU_TAG_PY3 --py-version 3 --processor gpu else echo "skipping sagemaker tests" fi @@ -109,5 +124,7 @@ phases: - cleanup-key-pairs # remove ecr image - - aws ecr batch-delete-image --repository-name $ECR_REPO --region $AWS_DEFAULT_REGION --image-ids imageTag=$CPU_TAG - - aws ecr batch-delete-image --repository-name $ECR_REPO --region $AWS_DEFAULT_REGION --image-ids imageTag=$GPU_TAG + - aws ecr batch-delete-image --repository-name $ECR_REPO --region $AWS_DEFAULT_REGION --image-ids imageTag=$CPU_TAG_PY2 + - aws ecr batch-delete-image --repository-name $ECR_REPO --region $AWS_DEFAULT_REGION --image-ids imageTag=$GPU_TAG_PY2 + - aws ecr batch-delete-image --repository-name $ECR_REPO --region $AWS_DEFAULT_REGION --image-ids imageTag=$CPU_TAG_PY3 + - aws ecr batch-delete-image --repository-name $ECR_REPO --region $AWS_DEFAULT_REGION --image-ids imageTag=$GPU_TAG_PY3 diff --git a/docker/__init__.py b/docker/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/docker/build_artifacts/__init__.py b/docker/build_artifacts/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/sagemaker_tensorflow_container/deep_learning_container.py b/docker/build_artifacts/deep_learning_container.py similarity index 100% rename from src/sagemaker_tensorflow_container/deep_learning_container.py rename to docker/build_artifacts/deep_learning_container.py diff --git a/docker/1.15.0/py2/dockerd-entrypoint.py b/docker/build_artifacts/dockerd-entrypoint.py similarity index 100% rename from docker/1.15.0/py2/dockerd-entrypoint.py rename to docker/build_artifacts/dockerd-entrypoint.py diff --git a/test/unit/test_deep_learning_containers.py b/test/unit/test_deep_learning_container.py similarity index 98% rename from test/unit/test_deep_learning_containers.py rename to test/unit/test_deep_learning_container.py index 8d6fe08e..7d5d7d86 100644 --- a/test/unit/test_deep_learning_containers.py +++ b/test/unit/test_deep_learning_container.py @@ -14,11 +14,10 @@ import unittest +from docker.build_artifacts import deep_learning_container as deep_learning_container_to_test import pytest import requests -from sagemaker_tensorflow_container import deep_learning_container as deep_learning_container_to_test - @pytest.fixture(name='fixture_valid_instance_id') def fixture_valid_instance_id(requests_mock):