Skip to content

Commit a7799e4

Browse files
knakadknakad
authored
change: fix test_processing for regions without m4.xlarge instances. (#1198)
1 parent 93dc590 commit a7799e4

File tree

1 file changed

+41
-26
lines changed

1 file changed

+41
-26
lines changed

‎tests/integ/test_processing.py

Lines changed: 41 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -107,7 +107,6 @@ def test_sklearn(sagemaker_session, sklearn_full_version, cpu_instance_type):
107107
instance_count=1,
108108
command=["python3"],
109109
sagemaker_session=sagemaker_session,
110-
max_runtime_in_seconds=3600, # TODO-reinvent-2019: REMOVE
111110
base_job_name="test-sklearn",
112111
)
113112

@@ -121,10 +120,12 @@ def test_sklearn(sagemaker_session, sklearn_full_version, cpu_instance_type):
121120
job_description = sklearn_processor.latest_job.describe()
122121

123122
assert len(job_description["ProcessingInputs"]) == 2
124-
assert job_description["ProcessingResources"] == {
125-
"ClusterConfig": {"InstanceCount": 1, "InstanceType": "ml.m4.xlarge", "VolumeSizeInGB": 30}
126-
}
127-
assert job_description["StoppingCondition"] == {"MaxRuntimeInSeconds": 3600}
123+
assert job_description["ProcessingResources"]["ClusterConfig"]["InstanceCount"] == 1
124+
assert (
125+
job_description["ProcessingResources"]["ClusterConfig"]["InstanceType"] == cpu_instance_type
126+
)
127+
assert job_description["ProcessingResources"]["ClusterConfig"]["VolumeSizeInGB"] == 30
128+
assert job_description["StoppingCondition"] == {"MaxRuntimeInSeconds": 86400}
128129
assert job_description["AppSpecification"]["ContainerEntrypoint"] == [
129130
"python3",
130131
"/opt/ml/processing/input/code/dummy_script.py",
@@ -191,9 +192,11 @@ def test_sklearn_with_customizations(
191192
assert job_description["ProcessingOutputConfig"]["KmsKeyId"] == output_kms_key
192193
assert job_description["ProcessingOutputConfig"]["Outputs"][0]["OutputName"] == "dummy_output"
193194

194-
assert job_description["ProcessingResources"] == {
195-
"ClusterConfig": {"InstanceCount": 1, "InstanceType": "ml.m4.xlarge", "VolumeSizeInGB": 100}
196-
}
195+
assert job_description["ProcessingResources"]["ClusterConfig"]["InstanceCount"] == 1
196+
assert (
197+
job_description["ProcessingResources"]["ClusterConfig"]["InstanceType"] == cpu_instance_type
198+
)
199+
assert job_description["ProcessingResources"]["ClusterConfig"]["VolumeSizeInGB"] == 100
197200

198201
assert job_description["AppSpecification"]["ContainerArguments"] == ["-v"]
199202
assert job_description["AppSpecification"]["ContainerEntrypoint"] == [
@@ -274,9 +277,11 @@ def test_sklearn_with_custom_default_bucket(
274277
assert job_description["ProcessingOutputConfig"]["KmsKeyId"] == output_kms_key
275278
assert job_description["ProcessingOutputConfig"]["Outputs"][0]["OutputName"] == "dummy_output"
276279

277-
assert job_description["ProcessingResources"] == {
278-
"ClusterConfig": {"InstanceCount": 1, "InstanceType": "ml.m4.xlarge", "VolumeSizeInGB": 100}
279-
}
280+
assert job_description["ProcessingResources"]["ClusterConfig"]["InstanceCount"] == 1
281+
assert (
282+
job_description["ProcessingResources"]["ClusterConfig"]["InstanceType"] == cpu_instance_type
283+
)
284+
assert job_description["ProcessingResources"]["ClusterConfig"]["VolumeSizeInGB"] == 100
280285

281286
assert job_description["AppSpecification"]["ContainerArguments"] == ["-v"]
282287
assert job_description["AppSpecification"]["ContainerEntrypoint"] == [
@@ -322,9 +327,11 @@ def test_sklearn_with_no_inputs_or_outputs(
322327

323328
assert job_description["ProcessingJobStatus"] == "Completed"
324329

325-
assert job_description["ProcessingResources"] == {
326-
"ClusterConfig": {"InstanceCount": 1, "InstanceType": "ml.m4.xlarge", "VolumeSizeInGB": 100}
327-
}
330+
assert job_description["ProcessingResources"]["ClusterConfig"]["InstanceCount"] == 1
331+
assert (
332+
job_description["ProcessingResources"]["ClusterConfig"]["InstanceType"] == cpu_instance_type
333+
)
334+
assert job_description["ProcessingResources"]["ClusterConfig"]["VolumeSizeInGB"] == 100
328335

329336
assert job_description["AppSpecification"]["ContainerArguments"] == ["-v"]
330337
assert job_description["AppSpecification"]["ContainerEntrypoint"] == [
@@ -397,9 +404,11 @@ def test_script_processor(sagemaker_session, image_uri, cpu_instance_type, outpu
397404
assert job_description["ProcessingOutputConfig"]["KmsKeyId"] == output_kms_key
398405
assert job_description["ProcessingOutputConfig"]["Outputs"][0]["OutputName"] == "dummy_output"
399406

400-
assert job_description["ProcessingResources"] == {
401-
"ClusterConfig": {"InstanceCount": 1, "InstanceType": "ml.m4.xlarge", "VolumeSizeInGB": 100}
402-
}
407+
assert job_description["ProcessingResources"]["ClusterConfig"]["InstanceCount"] == 1
408+
assert (
409+
job_description["ProcessingResources"]["ClusterConfig"]["InstanceType"] == cpu_instance_type
410+
)
411+
assert job_description["ProcessingResources"]["ClusterConfig"]["VolumeSizeInGB"] == 100
403412

404413
assert job_description["AppSpecification"]["ContainerArguments"] == ["-v"]
405414
assert job_description["AppSpecification"]["ContainerEntrypoint"] == [
@@ -445,9 +454,11 @@ def test_script_processor_with_no_inputs_or_outputs(
445454

446455
assert job_description["ProcessingJobStatus"] == "Completed"
447456

448-
assert job_description["ProcessingResources"] == {
449-
"ClusterConfig": {"InstanceCount": 1, "InstanceType": "ml.m4.xlarge", "VolumeSizeInGB": 100}
450-
}
457+
assert job_description["ProcessingResources"]["ClusterConfig"]["InstanceCount"] == 1
458+
assert (
459+
job_description["ProcessingResources"]["ClusterConfig"]["InstanceType"] == cpu_instance_type
460+
)
461+
assert job_description["ProcessingResources"]["ClusterConfig"]["VolumeSizeInGB"] == 100
451462

452463
assert job_description["AppSpecification"]["ContainerArguments"] == ["-v"]
453464
assert job_description["AppSpecification"]["ContainerEntrypoint"] == [
@@ -511,9 +522,11 @@ def test_processor(sagemaker_session, image_uri, cpu_instance_type, output_kms_k
511522
assert job_description["ProcessingOutputConfig"]["KmsKeyId"] == output_kms_key
512523
assert job_description["ProcessingOutputConfig"]["Outputs"][0]["OutputName"] == "dummy_output"
513524

514-
assert job_description["ProcessingResources"] == {
515-
"ClusterConfig": {"InstanceCount": 1, "InstanceType": "ml.m4.xlarge", "VolumeSizeInGB": 100}
516-
}
525+
assert job_description["ProcessingResources"]["ClusterConfig"]["InstanceCount"] == 1
526+
assert (
527+
job_description["ProcessingResources"]["ClusterConfig"]["InstanceType"] == cpu_instance_type
528+
)
529+
assert job_description["ProcessingResources"]["ClusterConfig"]["VolumeSizeInGB"] == 100
517530

518531
assert job_description["AppSpecification"]["ContainerArguments"] == ["-v"]
519532
assert job_description["AppSpecification"]["ContainerEntrypoint"] == [
@@ -580,9 +593,11 @@ def test_processor_with_custom_bucket(
580593
assert job_description["ProcessingOutputConfig"]["KmsKeyId"] == output_kms_key
581594
assert job_description["ProcessingOutputConfig"]["Outputs"][0]["OutputName"] == "dummy_output"
582595

583-
assert job_description["ProcessingResources"] == {
584-
"ClusterConfig": {"InstanceCount": 1, "InstanceType": "ml.m4.xlarge", "VolumeSizeInGB": 100}
585-
}
596+
assert job_description["ProcessingResources"]["ClusterConfig"]["InstanceCount"] == 1
597+
assert (
598+
job_description["ProcessingResources"]["ClusterConfig"]["InstanceType"] == cpu_instance_type
599+
)
600+
assert job_description["ProcessingResources"]["ClusterConfig"]["VolumeSizeInGB"] == 100
586601

587602
assert job_description["AppSpecification"]["ContainerArguments"] == ["-v"]
588603
assert job_description["AppSpecification"]["ContainerEntrypoint"] == [

0 commit comments

Comments
 (0)