Merge branch 'master' into fix-integ-tests

Author: ahsan-z-khan

.githooks/pre-push

@@ -0,0 +1,16 @@
+#!/bin/sh
+# this pre-push hook runs style checks and unit tests in python 3.6, 3.7, and 3.8 using tox.
+
+set -e
+
+TOX_PARALLEL_NO_SPINNER=1,
+PY_COLORS=0
+start_time=`date +%s`
+tox -e flake8,pylint,docstyle,black-check,twine --parallel all
+./ci-scripts/displaytime.sh 'flake8,pylint,docstyle,black-check,twine' $start_time
+start_time=`date +%s`
+tox -e sphinx,doc8 --parallel all
+./ci-scripts/displaytime.sh 'sphinx,doc8' $start_time
+start_time=`date +%s`
+tox -e py36,py37,py38 --parallel all -- tests/unit
+./ci-scripts/displaytime.sh 'py36,py37,py38 unit' $start_time

CHANGELOG.md

@@ -1,5 +1,17 @@
 # Changelog
 
+## v2.63.2 (2021-10-18)
+
+### Bug Fixes and Other Changes
+
+ * Update timeouts for integ tests from 20 to 40
+
+## v2.63.1 (2021-10-14)
+
+### Bug Fixes and Other Changes
+
+ * HF estimator attach modified to work with py38
+
 ## v2.63.0 (2021-10-13)
 
 ### Features

README.rst

@@ -154,6 +154,18 @@ You can also run them in parallel:
     tox -- -n auto tests/integ
 
 
+Git Hooks
+~~~~~~~~~
+
+to enable all git hooks in the .githooks directory, run these commands in the repository directory:
+
+::
+
+    find .git/hooks -type l -exec rm {} \;
+    find .githooks -type f -exec ln -sf ../../{} .git/hooks/ \;
+
+To enable an individual git hook, simply move it from the .githooks/ directory to the .git/hooks/ directory.
+
 Building Sphinx docs
 ~~~~~~~~~~~~~~~~~~~~
 

VERSION

@@ -1 +1 @@
-2.63.1.dev0
+2.63.3.dev0

src/sagemaker/huggingface/estimator.py

@@ -312,7 +312,7 @@ def _prepare_init_params_from_job_description(cls, job_details, model_channel_na
             framework_version = None
         else:
             framework, pt_or_tf = framework.split("-")
-            tag_pattern = re.compile("^(.*)-transformers(.*)-(cpu|gpu)-(py2|py3[67]?)$")
+            tag_pattern = re.compile(r"^(.*)-transformers(.*)-(cpu|gpu)-(py2|py3\d*)$")
             tag_match = tag_pattern.match(tag)
             pt_or_tf_version = tag_match.group(1)
             framework_version = tag_match.group(2)

tests/integ/__init__.py

@@ -19,8 +19,8 @@
 
 DATA_DIR = os.path.join(os.path.dirname(__file__), "..", "data")
 TRAINING_DEFAULT_TIMEOUT_MINUTES = 40
-TUNING_DEFAULT_TIMEOUT_MINUTES = 20
-TRANSFORM_DEFAULT_TIMEOUT_MINUTES = 20
+TUNING_DEFAULT_TIMEOUT_MINUTES = 40
+TRANSFORM_DEFAULT_TIMEOUT_MINUTES = 40
 AUTO_ML_DEFAULT_TIMEMOUT_MINUTES = 60
 
 # these regions have some p2 and p3 instances, but not enough for continuous testing

tests/unit/sagemaker/huggingface/test_estimator.py

@@ -253,12 +253,16 @@ def test_huggingface(
 
 
 def test_attach(
-    sagemaker_session, huggingface_training_version, huggingface_pytorch_training_version
+    sagemaker_session,
+    huggingface_training_version,
+    huggingface_pytorch_training_version,
+    huggingface_pytorch_training_py_version,
 ):
     training_image = (
         f"1.dkr.ecr.us-east-1.amazonaws.com/huggingface-pytorch-training:"
         f"{huggingface_pytorch_training_version}-"
-        f"transformers{huggingface_training_version}-gpu-py36-cu110-ubuntu18.04"
+        f"transformers{huggingface_training_version}-gpu-"
+        f"{huggingface_pytorch_training_py_version}-cu110-ubuntu20.04"
     )
     returned_job_description = {
         "AlgorithmSpecification": {"TrainingInputMode": "File", "TrainingImage": training_image},
@@ -290,7 +294,7 @@ def test_attach(
 
     estimator = HuggingFace.attach(training_job_name="neo", sagemaker_session=sagemaker_session)
     assert estimator.latest_training_job.job_name == "neo"
-    assert estimator.py_version == "py36"
+    assert estimator.py_version == huggingface_pytorch_training_py_version
     assert estimator.framework_version == huggingface_training_version
     assert estimator.pytorch_version == huggingface_pytorch_training_version
     assert estimator.role == "arn:aws:iam::366:role/SageMakerRole"