Skip to content

Commit d3be091

Browse files
author
Liu
committed
feature: disable default rule
1 parent b383a31 commit d3be091

13 files changed

+5
-121
lines changed

‎src/sagemaker/estimator.py

Lines changed: 5 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -914,29 +914,25 @@ def _prepare_collection_configs(self):
914914
def _prepare_profiler_for_training(self):
915915
"""Set necessary values and do basic validations in profiler config and profiler rules.
916916
917-
When user explicitly set rules to an empty list, default profiler rule won't be enabled.
918-
Default profiler rule will be enabled in supported regions when either:
919-
1. user doesn't specify any rules, i.e., rules=None; or
920-
2. user only specify debugger rules, i.e., rules=[Rule.sagemaker(...)]
917+
No default profiler rule will be used. The user needs to specify rules explicitly
921918
"""
922919
if self.disable_profiler:
923920
if self.profiler_config and self.profiler_config.disable_profiler == False:
924-
raise RuntimeError("profiler_config cannot be set when disable_profiler is True.")
921+
raise RuntimeError("profiler_config.disable_profiler cannot be False when disable_profiler is True.")
925922
if self.profiler_rules:
926923
raise RuntimeError("ProfilerRule cannot be set when disable_profiler is True.")
927924
elif _region_supports_profiler(self.sagemaker_session.boto_region_name):
928925
if self.profiler_config is None:
929926
self.profiler_config = ProfilerConfig(s3_output_path=self.output_path)
930927
if self.rules is None or (self.rules and not self.profiler_rules):
931-
self.profiler_rules = [get_default_profiler_rule()]
928+
self.profiler_rules = []
932929

933930
if self.profiler_config and not self.profiler_config.s3_output_path:
934931
self.profiler_config.s3_output_path = self.output_path
935932

936933
self.profiler_rule_configs = self._prepare_profiler_rules()
937934
# if profiler_config is still None, it means the job has profiler disabled
938935
if self.profiler_config is None:
939-
# self.profiler_config = ProfilerConfig(disable_profiler=True)
940936
self.profiler_config = ProfilerConfig(
941937
s3_output_path=self.output_path, disable_profiler=True
942938
)
@@ -1861,8 +1857,8 @@ def enable_default_profiling(self):
18611857
else:
18621858
self.profiler_config = ProfilerConfig(s3_output_path=self.output_path)
18631859

1864-
self.profiler_rules = [get_default_profiler_rule()]
1865-
self.profiler_rule_configs = self._prepare_profiler_rules()
1860+
self.profiler_rules = []
1861+
self.profiler_rule_configs = []
18661862

18671863
_TrainingJob.update(
18681864
self, self.profiler_rule_configs, self.profiler_config._to_request_dict()

‎tests/unit/sagemaker/huggingface/test_estimator.py

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -142,13 +142,6 @@ def _create_train_job(version, base_framework_version):
142142
"CollectionConfigurations": [],
143143
"S3OutputPath": "s3://{}/".format(BUCKET_NAME),
144144
},
145-
"profiler_rule_configs": [
146-
{
147-
"RuleConfigurationName": "ProfilerReport-1510006209",
148-
"RuleEvaluatorImage": "503895931360.dkr.ecr.us-east-1.amazonaws.com/sagemaker-debugger-rules:latest",
149-
"RuleParameters": {"rule_to_invoke": "ProfilerReport"},
150-
}
151-
],
152145
"profiler_config": {
153146
"DisableProfiler": False,
154147
"S3OutputPath": "s3://{}/".format(BUCKET_NAME),

‎tests/unit/sagemaker/tensorflow/test_estimator.py

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -135,13 +135,6 @@ def _create_train_job(tf_version, horovod=False, ps=False, py_version="py2", smd
135135
"metric_definitions": None,
136136
"environment": None,
137137
"experiment_config": None,
138-
"profiler_rule_configs": [
139-
{
140-
"RuleConfigurationName": "ProfilerReport-1510006209",
141-
"RuleEvaluatorImage": "895741380848.dkr.ecr.us-west-2.amazonaws.com/sagemaker-debugger-rules:latest",
142-
"RuleParameters": {"rule_to_invoke": "ProfilerReport"},
143-
}
144-
],
145138
"profiler_config": {
146139
"DisableProfiler": False,
147140
"S3OutputPath": "s3://{}/".format(BUCKET_NAME),

‎tests/unit/sagemaker/training_compiler/test_huggingface_pytorch_compiler.py

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -144,13 +144,6 @@ def _create_train_job(
144144
"CollectionConfigurations": [],
145145
"S3OutputPath": "s3://{}/".format(BUCKET_NAME),
146146
},
147-
"profiler_rule_configs": [
148-
{
149-
"RuleConfigurationName": "ProfilerReport-1510006209",
150-
"RuleEvaluatorImage": "503895931360.dkr.ecr.us-east-1.amazonaws.com/sagemaker-debugger-rules:latest",
151-
"RuleParameters": {"rule_to_invoke": "ProfilerReport"},
152-
}
153-
],
154147
"profiler_config": {
155148
"DisableProfiler": False,
156149
"S3OutputPath": "s3://{}/".format(BUCKET_NAME),

‎tests/unit/sagemaker/training_compiler/test_huggingface_tensorflow_compiler.py

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -142,13 +142,6 @@ def _create_train_job(
142142
"CollectionConfigurations": [],
143143
"S3OutputPath": "s3://{}/".format(BUCKET_NAME),
144144
},
145-
"profiler_rule_configs": [
146-
{
147-
"RuleConfigurationName": "ProfilerReport-1510006209",
148-
"RuleEvaluatorImage": "503895931360.dkr.ecr.us-east-1.amazonaws.com/sagemaker-debugger-rules:latest",
149-
"RuleParameters": {"rule_to_invoke": "ProfilerReport"},
150-
}
151-
],
152145
"profiler_config": {
153146
"DisableProfiler": False,
154147
"S3OutputPath": "s3://{}/".format(BUCKET_NAME),

‎tests/unit/sagemaker/training_compiler/test_tensorflow_compiler.py

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -144,13 +144,6 @@ def _create_train_job(framework_version, instance_type, training_compiler_config
144144
"CollectionConfigurations": [],
145145
"S3OutputPath": "s3://{}/".format(BUCKET_NAME),
146146
},
147-
"profiler_rule_configs": [
148-
{
149-
"RuleConfigurationName": "ProfilerReport-1510006209",
150-
"RuleEvaluatorImage": "503895931360.dkr.ecr.us-east-1.amazonaws.com/sagemaker-debugger-rules:latest",
151-
"RuleParameters": {"rule_to_invoke": "ProfilerReport"},
152-
}
153-
],
154147
"profiler_config": {
155148
"DisableProfiler": False,
156149
"S3OutputPath": "s3://{}/".format(BUCKET_NAME),

‎tests/unit/test_chainer.py

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -150,13 +150,6 @@ def _create_train_job(version, py_version):
150150
"CollectionConfigurations": [],
151151
"S3OutputPath": "s3://{}/".format(BUCKET_NAME),
152152
},
153-
"profiler_rule_configs": [
154-
{
155-
"RuleConfigurationName": "ProfilerReport-1510006209",
156-
"RuleEvaluatorImage": "895741380848.dkr.ecr.us-west-2.amazonaws.com/sagemaker-debugger-rules:latest",
157-
"RuleParameters": {"rule_to_invoke": "ProfilerReport"},
158-
}
159-
],
160153
"profiler_config": {
161154
"DisableProfiler": False,
162155
"S3OutputPath": "s3://{}/".format(BUCKET_NAME),

‎tests/unit/test_estimator.py

Lines changed: 0 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -574,13 +574,6 @@ def test_framework_without_debugger_and_profiler(time, sagemaker_session):
574574
"DisableProfiler": False,
575575
"S3OutputPath": "s3://{}/".format(BUCKET_NAME),
576576
}
577-
assert args["profiler_rule_configs"] == [
578-
{
579-
"RuleConfigurationName": "ProfilerReport-1510006209",
580-
"RuleEvaluatorImage": "895741380848.dkr.ecr.us-west-2.amazonaws.com/sagemaker-debugger-rules:latest",
581-
"RuleParameters": {"rule_to_invoke": "ProfilerReport"},
582-
}
583-
]
584577

585578

586579
def test_framework_with_debugger_and_profiler_rules(sagemaker_session):
@@ -715,13 +708,6 @@ def test_framework_with_profiler_config_without_s3_output_path(time, sagemaker_s
715708
"S3OutputPath": "s3://{}/".format(BUCKET_NAME),
716709
"ProfilingIntervalInMilliseconds": 1000,
717710
}
718-
assert args["profiler_rule_configs"] == [
719-
{
720-
"RuleConfigurationName": "ProfilerReport-1510006209",
721-
"RuleEvaluatorImage": "895741380848.dkr.ecr.us-west-2.amazonaws.com/sagemaker-debugger-rules:latest",
722-
"RuleParameters": {"rule_to_invoke": "ProfilerReport"},
723-
}
724-
]
725711

726712

727713
@pytest.mark.parametrize("region", PROFILER_UNSUPPORTED_REGIONS)
@@ -933,13 +919,6 @@ def test_framework_with_enabling_default_profiling(
933919
"DisableProfiler": False,
934920
"S3OutputPath": "s3://{}/".format(BUCKET_NAME),
935921
}
936-
assert args["profiler_rule_configs"] == [
937-
{
938-
"RuleConfigurationName": "ProfilerReport-1510006209",
939-
"RuleEvaluatorImage": "895741380848.dkr.ecr.us-west-2.amazonaws.com/sagemaker-debugger-rules:latest",
940-
"RuleParameters": {"rule_to_invoke": "ProfilerReport"},
941-
}
942-
]
943922

944923

945924
@patch("time.time", return_value=TIME)
@@ -967,13 +946,6 @@ def test_framework_with_enabling_default_profiling_with_existed_s3_output_path(
967946
"DisableProfiler": False,
968947
"S3OutputPath": "s3://custom/",
969948
}
970-
assert args["profiler_rule_configs"] == [
971-
{
972-
"RuleConfigurationName": "ProfilerReport-1510006209",
973-
"RuleEvaluatorImage": "895741380848.dkr.ecr.us-west-2.amazonaws.com/sagemaker-debugger-rules:latest",
974-
"RuleParameters": {"rule_to_invoke": "ProfilerReport"},
975-
}
976-
]
977949

978950

979951
def test_framework_with_disabling_profiling_when_profiler_is_already_disabled(
@@ -2643,13 +2615,6 @@ def test_unsupported_type_in_dict():
26432615
"input_mode": "File",
26442616
"output_config": {"S3OutputPath": OUTPUT_PATH},
26452617
"profiler_config": {"DisableProfiler": False, "S3OutputPath": OUTPUT_PATH},
2646-
"profiler_rule_configs": [
2647-
{
2648-
"RuleConfigurationName": "ProfilerReport-1510006209",
2649-
"RuleEvaluatorImage": "895741380848.dkr.ecr.us-west-2.amazonaws.com/sagemaker-debugger-rules:latest",
2650-
"RuleParameters": {"rule_to_invoke": "ProfilerReport"},
2651-
}
2652-
],
26532618
"resource_config": {
26542619
"InstanceCount": INSTANCE_COUNT,
26552620
"InstanceType": INSTANCE_TYPE,

‎tests/unit/test_mxnet.py

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -159,13 +159,6 @@ def _get_train_args(job_name):
159159
"CollectionConfigurations": [],
160160
"S3OutputPath": "s3://{}/".format(BUCKET_NAME),
161161
},
162-
"profiler_rule_configs": [
163-
{
164-
"RuleConfigurationName": "ProfilerReport-1510006209",
165-
"RuleEvaluatorImage": "520713654638.dkr.ecr.us-west-2.amazonaws.com/sagemaker-mxnet:1.4.0-cpu-py3",
166-
"RuleParameters": {"rule_to_invoke": "ProfilerReport"},
167-
}
168-
],
169162
"profiler_config": {
170163
"DisableProfiler": False,
171164
"S3OutputPath": "s3://{}/".format(BUCKET_NAME),

‎tests/unit/test_pytorch.py

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -157,13 +157,6 @@ def _create_train_job(version, py_version):
157157
"CollectionConfigurations": [],
158158
"S3OutputPath": "s3://{}/".format(BUCKET_NAME),
159159
},
160-
"profiler_rule_configs": [
161-
{
162-
"RuleConfigurationName": "ProfilerReport-1510006209",
163-
"RuleEvaluatorImage": "895741380848.dkr.ecr.us-west-2.amazonaws.com/sagemaker-debugger-rules:latest",
164-
"RuleParameters": {"rule_to_invoke": "ProfilerReport"},
165-
}
166-
],
167160
"profiler_config": {
168161
"DisableProfiler": False,
169162
"S3OutputPath": "s3://{}/".format(BUCKET_NAME),

‎tests/unit/test_rl.py

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -152,13 +152,6 @@ def _create_train_job(toolkit, toolkit_version, framework):
152152
"CollectionConfigurations": [],
153153
"S3OutputPath": "s3://{}/".format(BUCKET_NAME),
154154
},
155-
"profiler_rule_configs": [
156-
{
157-
"RuleConfigurationName": "ProfilerReport-1510006209",
158-
"RuleEvaluatorImage": "895741380848.dkr.ecr.us-west-2.amazonaws.com/sagemaker-debugger-rules:latest",
159-
"RuleParameters": {"rule_to_invoke": "ProfilerReport"},
160-
}
161-
],
162155
"profiler_config": {
163156
"DisableProfiler": False,
164157
"S3OutputPath": "s3://{}/".format(BUCKET_NAME),

‎tests/unit/test_sklearn.py

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -139,13 +139,6 @@ def _create_train_job(version):
139139
"CollectionConfigurations": [],
140140
"S3OutputPath": "s3://{}/".format(BUCKET_NAME),
141141
},
142-
"profiler_rule_configs": [
143-
{
144-
"RuleConfigurationName": "ProfilerReport-1510006209",
145-
"RuleEvaluatorImage": "895741380848.dkr.ecr.us-west-2.amazonaws.com/sagemaker-debugger-rules:latest",
146-
"RuleParameters": {"rule_to_invoke": "ProfilerReport"},
147-
}
148-
],
149142
"profiler_config": {
150143
"DisableProfiler": False,
151144
"S3OutputPath": "s3://{}/".format(BUCKET_NAME),

‎tests/unit/test_xgboost.py

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -154,13 +154,6 @@ def _create_train_job(version, instance_count=1, instance_type="ml.c4.4xlarge"):
154154
"CollectionConfigurations": [],
155155
"S3OutputPath": "s3://{}/".format(BUCKET_NAME),
156156
},
157-
"profiler_rule_configs": [
158-
{
159-
"RuleConfigurationName": "ProfilerReport-1510006209",
160-
"RuleEvaluatorImage": "895741380848.dkr.ecr.us-west-2.amazonaws.com/sagemaker-debugger-rules:latest",
161-
"RuleParameters": {"rule_to_invoke": "ProfilerReport"},
162-
}
163-
],
164157
"profiler_config": {
165158
"DisableProfiler": False,
166159
"S3OutputPath": "s3://{}/".format(BUCKET_NAME),

0 commit comments

Comments
 (0)