From 43870e5f99b1261705cb5292ba2631c3e5f926fb Mon Sep 17 00:00:00 2001 From: Chuyang Deng Date: Mon, 4 May 2020 16:44:17 -0700 Subject: [PATCH 1/3] infra: support Python 3.7 --- README.rst | 5 +++-- buildspec-localmodetests.yml | 4 ++-- buildspec-release.yml | 4 ++-- buildspec-unittests.yml | 4 ++-- buildspec.yml | 6 +++--- setup.py | 1 + src/sagemaker/tensorflow/estimator.py | 11 +++++++---- tox.ini | 2 +- 8 files changed, 21 insertions(+), 16 deletions(-) diff --git a/README.rst b/README.rst index 6f89b1a6a4..a6a75d8a8e 100644 --- a/README.rst +++ b/README.rst @@ -90,6 +90,7 @@ SageMaker Python SDK is tested on: - Python 2.7 - Python 3.6 +- Python 3.7 AWS Permissions ~~~~~~~~~~~~~~~ @@ -117,8 +118,8 @@ You can install the libraries needed to run the tests by running :code:`pip inst We run unit tests with tox, which is a program that lets you run unit tests for multiple Python versions, and also make sure the -code fits our style guidelines. We run tox with Python 2.7 and 3.6, so to run unit tests -with the same configuration we do, you'll need to have interpreters for Python 2.7 and Python 3.6 installed. +code fits our style guidelines. We run tox with Python 2.7, 3.6 and 3.7, so to run unit tests +with the same configuration we do, you'll need to have interpreters for Python 2.7, Python 3.6 and Python 3.7 installed. To run the unit tests with tox, run: diff --git a/buildspec-localmodetests.yml b/buildspec-localmodetests.yml index b775ac2062..d059babfd4 100644 --- a/buildspec-localmodetests.yml +++ b/buildspec-localmodetests.yml @@ -11,5 +11,5 @@ phases: # local mode tests - start_time=`date +%s` - - execute-command-if-has-matching-changes "tox -e py27,py36 -- tests/integ -m local_mode --durations 50" "tests/integ" "tests/data" "tests/conftest.py" "tests/__init__.py" "src/*.py" "setup.py" "setup.cfg" "buildspec-localmodetests.yml" - - ./ci-scripts/displaytime.sh 'py27,py36 local mode' $start_time + - execute-command-if-has-matching-changes "tox -e py27,py37 -- tests/integ -m local_mode --durations 50" "tests/integ" "tests/data" "tests/conftest.py" "tests/__init__.py" "src/*.py" "setup.py" "setup.cfg" "buildspec-localmodetests.yml" + - ./ci-scripts/displaytime.sh 'py27,py36,py37 local mode' $start_time diff --git a/buildspec-release.yml b/buildspec-release.yml index 7bcea98ee3..bb3bfcdd34 100644 --- a/buildspec-release.yml +++ b/buildspec-release.yml @@ -18,10 +18,10 @@ phases: # run unit tests - AWS_ACCESS_KEY_ID= AWS_SECRET_ACCESS_KEY= AWS_SESSION_TOKEN= AWS_CONTAINER_CREDENTIALS_RELATIVE_URI= AWS_DEFAULT_REGION= - tox -e py27,py36 -- tests/unit + tox -e py27,py36,py37 -- tests/unit # run a subset of the integration tests - - IGNORE_COVERAGE=- tox -e py36 -- tests/integ -m canary_quick -n 64 --boxed --reruns 2 + - IGNORE_COVERAGE=- tox -e py37 -- tests/integ -m canary_quick -n 64 --boxed --reruns 2 # generate the distribution package - python3 setup.py sdist diff --git a/buildspec-unittests.yml b/buildspec-unittests.yml index b2b977f0a0..41a6f68621 100644 --- a/buildspec-unittests.yml +++ b/buildspec-unittests.yml @@ -18,5 +18,5 @@ phases: - start_time=`date +%s` - AWS_ACCESS_KEY_ID= AWS_SECRET_ACCESS_KEY= AWS_SESSION_TOKEN= AWS_CONTAINER_CREDENTIALS_RELATIVE_URI= AWS_DEFAULT_REGION= - tox -e py36,py27 --parallel all -- tests/unit - - ./ci-scripts/displaytime.sh 'py36,py27 unit' $start_time + tox -e py27,py36,py37 --parallel all -- tests/unit + - ./ci-scripts/displaytime.sh 'py27,py36,py37 unit' $start_time diff --git a/buildspec.yml b/buildspec.yml index 87b9ad3a10..6cecfba123 100644 --- a/buildspec.yml +++ b/buildspec.yml @@ -11,13 +11,13 @@ phases: # run integration tests - start_time=`date +%s` - - execute-command-if-has-matching-changes "python3 -u ci-scripts/queue_build.py" "tests/integ" "tests/scripts" "tests/data" "tests/conftest.py" "tests/__init__.py" "src/*.py" "setup.py" "setup.cfg" "buildspec.yml" + - execute-command-if-has-matching-changes "python3.7 -u ci-scripts/queue_build.py" "tests/integ" "tests/scripts" "tests/data" "tests/conftest.py" "tests/__init__.py" "src/*.py" "setup.py" "setup.cfg" "buildspec.yml" - ./ci-scripts/displaytime.sh 'build queue' $start_time - start_time=`date +%s` - | - execute-command-if-has-matching-changes "env -u AWS_DEFAULT_REGION tox -e py36 -- tests/integ -m \"not local_mode\" -n 512 --reruns 3 --reruns-delay 5 --durations 50 --boto-config '{\"region_name\": \"us-east-2\"}'" "tests/integ" "tests/scripts" "tests/data" "tests/conftest.py" "tests/__init__.py" "src/*.py" "setup.py" "setup.cfg" "buildspec.yml" - - ./ci-scripts/displaytime.sh 'py36 tests/integ' $start_time + execute-command-if-has-matching-changes "env -u AWS_DEFAULT_REGION tox -e py37 -- tests/integ -m \"not local_mode\" -n 512 --reruns 3 --reruns-delay 5 --durations 50 --boto-config '{\"region_name\": \"us-east-2\"}'" "tests/integ" "tests/scripts" "tests/data" "tests/conftest.py" "tests/__init__.py" "src/*.py" "setup.py" "setup.cfg" "buildspec.yml" + - ./ci-scripts/displaytime.sh 'py37 tests/integ' $start_time post_build: finally: diff --git a/setup.py b/setup.py index 0951a8babd..0c8ffcebd0 100644 --- a/setup.py +++ b/setup.py @@ -101,6 +101,7 @@ def read_version(): "Programming Language :: Python", "Programming Language :: Python :: 2.7", "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", ], install_requires=required_packages, extras_require=extras, diff --git a/src/sagemaker/tensorflow/estimator.py b/src/sagemaker/tensorflow/estimator.py index ef5a4c0719..47440ba21d 100644 --- a/src/sagemaker/tensorflow/estimator.py +++ b/src/sagemaker/tensorflow/estimator.py @@ -73,10 +73,13 @@ def __init__(self, estimator, logdir=None): @staticmethod def _cmd_exists(cmd): """Placeholder docstring""" - return any( - os.access(os.path.join(path, cmd), os.X_OK) - for path in os.environ["PATH"].split(os.pathsep) - ) + for path in os.environ["PATH"].split(os.pathsep): + try: + if os.access(os.path.join(path, cmd), os.X_OK): + return True + except StopIteration: + return False + return False @staticmethod def _sync_directories(from_directory, to_directory): diff --git a/tox.ini b/tox.ini index 979b211cdc..78cd0f1d4a 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,7 @@ # and then run "tox" from this directory. [tox] -envlist = black-format,flake8,pylint,twine,sphinx,doc8,py27,py36 +envlist = black-format,flake8,pylint,twine,sphinx,doc8,py27,py36,py37 skip_missing_interpreters = False From 2f887f1de8d64033abaf4c96214cad63a3f5fe47 Mon Sep 17 00:00:00 2001 From: Chuyang Deng Date: Mon, 4 May 2020 16:47:14 -0700 Subject: [PATCH 2/3] release build integ tests run against py36 --- buildspec-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildspec-release.yml b/buildspec-release.yml index bb3bfcdd34..31814ced39 100644 --- a/buildspec-release.yml +++ b/buildspec-release.yml @@ -21,7 +21,7 @@ phases: tox -e py27,py36,py37 -- tests/unit # run a subset of the integration tests - - IGNORE_COVERAGE=- tox -e py37 -- tests/integ -m canary_quick -n 64 --boxed --reruns 2 + - IGNORE_COVERAGE=- tox -e py36 -- tests/integ -m canary_quick -n 64 --boxed --reruns 2 # generate the distribution package - python3 setup.py sdist From 2214f75cab652b7465ccd254539c4558856abd91 Mon Sep 17 00:00:00 2001 From: Chuyang Deng Date: Tue, 5 May 2020 10:44:32 -0700 Subject: [PATCH 3/3] remove py36 in localmode buildspec ci-script --- buildspec-localmodetests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildspec-localmodetests.yml b/buildspec-localmodetests.yml index d059babfd4..3e82906546 100644 --- a/buildspec-localmodetests.yml +++ b/buildspec-localmodetests.yml @@ -12,4 +12,4 @@ phases: # local mode tests - start_time=`date +%s` - execute-command-if-has-matching-changes "tox -e py27,py37 -- tests/integ -m local_mode --durations 50" "tests/integ" "tests/data" "tests/conftest.py" "tests/__init__.py" "src/*.py" "setup.py" "setup.cfg" "buildspec-localmodetests.yml" - - ./ci-scripts/displaytime.sh 'py27,py36,py37 local mode' $start_time + - ./ci-scripts/displaytime.sh 'py27,py37 local mode' $start_time