Skip to content

feat: add Tensorflow and Pytorch version for SM Training Compiler and expand to regular regions #3045

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 8 commits into from
Apr 19, 2022
Merged

feat: add Tensorflow and Pytorch version for SM Training Compiler and expand to regular regions #3045

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
4f31c86
update tf version for training compiler and expand to regular regions
Apr 4, 2022
796f2cd
removing gov cloud regions and making pytorch container cuda-11.3
Apr 13, 2022
f34d2ac
add unit tests for tensorflow and update version for pytorch. Added a…
Apr 13, 2022
b176fdc
nit:fixing alignment
Apr 13, 2022
86f3594
fixing tf and pt unit tests
Apr 14, 2022
b130ad4
fixing black-check
Apr 14, 2022
08058f9
fix integ tests for tensorflow
Apr 18, 2022
43fdd48
updating names to trcomp
Apr 18, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
65 changes: 64 additions & 1 deletion src/sagemaker/image_uri_config/huggingface-training-compiler.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,8 @@
"training": {
"processors": ["gpu"],
"version_aliases": {
"4.11": "4.11.0"
"4.11": "4.11.0",
"4.17": "4.17.0"
},
"versions": {
"4.11.0": {
Expand Down Expand Up @@ -32,6 +33,68 @@
"repository": "huggingface-tensorflow-trcomp-training",
"container_version": {"gpu":"cu112-ubuntu18.04"}
}
},
"4.17.0": {
"version_aliases": {
"pytorch1.10": "pytorch1.10.2",
"tensorflow2.6": "tensorflow2.6.3"
},
"pytorch1.10.2": {
"py_versions": ["py38"],
"registries": {
"af-south-1": "626614931356",
"ap-east-1": "871362719292",
"ap-northeast-1": "763104351884",
"ap-northeast-2": "763104351884",
"ap-northeast-3": "364406365360",
"ap-south-1": "763104351884",
"ap-southeast-1": "763104351884",
"ap-southeast-2": "763104351884",
"ca-central-1": "763104351884",
"eu-central-1": "763104351884",
"eu-north-1": "763104351884",
"eu-south-1": "692866216735",
"eu-west-1": "763104351884",
"eu-west-2": "763104351884",
"eu-west-3": "763104351884",
"me-south-1": "217643126080",
"sa-east-1": "763104351884",
"us-east-1": "763104351884",
"us-east-2": "763104351884",
"us-west-1": "763104351884",
"us-west-2": "763104351884"
},
"repository": "huggingface-pytorch-trcomp-training",
"container_version": {"gpu":"cu113-ubuntu20.04"}
},
"tensorflow2.6.3": {
"py_versions": ["py38"],
"registries": {
"af-south-1": "626614931356",
"ap-east-1": "871362719292",
"ap-northeast-1": "763104351884",
"ap-northeast-2": "763104351884",
"ap-northeast-3": "364406365360",
"ap-south-1": "763104351884",
"ap-southeast-1": "763104351884",
"ap-southeast-2": "763104351884",
"ca-central-1": "763104351884",
"eu-central-1": "763104351884",
"eu-north-1": "763104351884",
"eu-south-1": "692866216735",
"eu-west-1": "763104351884",
"eu-west-2": "763104351884",
"eu-west-3": "763104351884",
"me-south-1": "217643126080",
"sa-east-1": "763104351884",
"us-east-1": "763104351884",
"us-east-2": "763104351884",
"us-west-1": "763104351884",
"us-west-2": "763104351884"
},
"repository": "huggingface-tensorflow-trcomp-training",
"container_version": {"gpu":"cu112-ubuntu20.04"}
}
}
}
}
Expand Down
9 changes: 9 additions & 0 deletions tests/conftest.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -248,6 +248,15 @@ def huggingface_training_compiler_tensorflow_version(huggingface_training_compil
)[0]


@pytest.fixture(scope="module")
def huggingface_training_compiler_py_version(huggingface_training_compiler_tensorflow_version):
return (
"py37"
if Version(huggingface_training_compiler_tensorflow_version) < Version("2.6")
else "py38"
)


@pytest.fixture(scope="module")
def huggingface_pytorch_latest_training_py_version(huggingface_training_pytorch_latest_version):
return (
Expand Down
20 changes: 20 additions & 0 deletions tests/integ/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -158,6 +158,26 @@
]
# TODO: SM Training Compiler team to add all supported regions.
TRAINING_COMPILER_SUPPORTED_REGIONS = [
"af-south-1",
"ap-east-1",
"ap-northeast-1",
"ap-northeast-2",
"ap-northeast-3",
"ap-south-1",
"ap-southeast-1",
"ap-southeast-2",
"ca-central-1",
"eu-central-1",
"eu-north-1",
"eu-south-1",
"eu-west-1",
"eu-west-2",
"eu-west-3",
"me-south-1",
"sa-east-1",
"us-east-1",
"us-east-2",
"us-west-1",
"us-west-2",
]
# Data parallelism need to be tested with p3.16xlarge.
Expand Down
2 changes: 1 addition & 1 deletion tests/integ/test_training_compiler.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,7 @@ def test_huggingface_tensorflow(
data_path = os.path.join(DATA_DIR, "huggingface")

hf = HuggingFace(
py_version="py37",
py_version="py38",
entry_point=os.path.join(data_path, "run_tf.py"),
role="SageMakerRole",
transformers_version=huggingface_training_compiler_latest_version,
Expand Down
32 changes: 16 additions & 16 deletions ...ing_compiler/test_huggingface_compiler.py → ...iler/test_huggingface_pytorch_compiler.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -158,9 +158,9 @@ def test_unsupported_BYOC(
):
byoc = (
"1.dkr.ecr.us-east-1.amazonaws.com/huggingface-pytorch-trcomp-training:"
"1.9.0-"
"transformers4.10.2-gpu-"
"py38-cu111-ubuntu20.04"
"1.10.2-"
"transformers4.17.0-gpu-"
"py38-cu113-ubuntu20.04"
)
with pytest.raises(ValueError):
HuggingFace(
Expand Down Expand Up @@ -451,9 +451,9 @@ def test_attach(
):
training_image = (
"1.dkr.ecr.us-east-1.amazonaws.com/huggingface-pytorch-trcomp-training:"
"1.9.0-"
"transformers4.10.2-gpu-"
"py38-cu111-ubuntu20.04"
"1.10.2-"
"transformers4.17.0-gpu-"
"py38-cu113-ubuntu20.04"
)
returned_job_description = {
"AlgorithmSpecification": {"TrainingInputMode": "File", "TrainingImage": training_image},
Expand All @@ -462,7 +462,7 @@ def test_attach(
"sagemaker_program": '"iris-dnn-classifier.py"',
"sagemaker_s3_uri_training": '"sagemaker-3/integ-test-data/tf_iris"',
"sagemaker_container_log_level": '"logging.INFO"',
"sagemaker_job_name": '"hopper"',
"sagemaker_job_name": '"trcomp"',
"training_steps": "100",
"sagemaker_region": '"us-east-1"',
TrainingCompilerConfig.HP_ENABLE_COMPILER: json.dumps(compiler_enabled),
Expand All @@ -475,27 +475,27 @@ def test_attach(
"InstanceType": "ml.p3.2xlarge",
},
"StoppingCondition": {"MaxRuntimeInSeconds": 24 * 60 * 60},
"TrainingJobName": "hopper",
"TrainingJobName": "trcomp",
"TrainingJobStatus": "Completed",
"TrainingJobArn": "arn:aws:sagemaker:us-west-2:336:training-job/hopper",
"OutputDataConfig": {"KmsKeyId": "", "S3OutputPath": "s3://place/output/hopper"},
"TrainingJobArn": "arn:aws:sagemaker:us-west-2:336:training-job/trcomp",
"OutputDataConfig": {"KmsKeyId": "", "S3OutputPath": "s3://place/output/trcomp"},
"TrainingJobOutput": {"S3TrainingJobOutput": "s3://here/output.tar.gz"},
}
sagemaker_session.sagemaker_client.describe_training_job = Mock(
name="describe_training_job", return_value=returned_job_description
)

estimator = HuggingFace.attach(training_job_name="hopper", sagemaker_session=sagemaker_session)
assert estimator.latest_training_job.job_name == "hopper"
estimator = HuggingFace.attach(training_job_name="trcomp", sagemaker_session=sagemaker_session)
assert estimator.latest_training_job.job_name == "trcomp"
assert estimator.py_version == "py38"
assert estimator.framework_version == "4.10.2"
assert estimator.pytorch_version == "1.9.0"
assert estimator.framework_version == "4.17.0"
assert estimator.pytorch_version == "1.10.2"
assert estimator.role == "arn:aws:iam::366:role/SageMakerRole"
assert estimator.instance_count == 1
assert estimator.max_run == 24 * 60 * 60
assert estimator.input_mode == "File"
assert estimator.base_job_name == "hopper"
assert estimator.output_path == "s3://place/output/hopper"
assert estimator.base_job_name == "trcomp"
assert estimator.output_path == "s3://place/output/trcomp"
assert estimator.output_kms_key == ""
assert estimator.hyperparameters()["training_steps"] == "100"
assert estimator.hyperparameters()[TrainingCompilerConfig.HP_ENABLE_COMPILER] == json.dumps(
Expand Down
Loading