From 57c3d52b0ef157ba520eaf49b69f2819df9e3955 Mon Sep 17 00:00:00 2001 From: Michael Date: Fri, 28 Apr 2023 11:56:52 -0700 Subject: [PATCH 1/3] fix: Update Clarify SHAPConfig baseline to allow JSON structures (#3804) Co-authored-by: Michael Trinh --- src/sagemaker/clarify.py | 6 ++++-- tests/unit/test_clarify.py | 22 ++++++++++++++-------- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/src/sagemaker/clarify.py b/src/sagemaker/clarify.py index 2f7f3dc53d..14bb675681 100644 --- a/src/sagemaker/clarify.py +++ b/src/sagemaker/clarify.py @@ -94,6 +94,8 @@ {object: object}, ) ], + # Arbitrary JSON object as baseline + {object: object}, ), SchemaOptional("num_clusters"): int, SchemaOptional("use_logit"): bool, @@ -1211,7 +1213,7 @@ class SHAPConfig(ExplainabilityConfig): def __init__( self, - baseline: Optional[Union[str, List]] = None, + baseline: Optional[Union[str, List, Dict]] = None, num_samples: Optional[int] = None, agg_method: Optional[str] = None, use_logit: bool = False, @@ -1224,7 +1226,7 @@ def __init__( """Initializes config for SHAP analysis. Args: - baseline (None or str or list): `Baseline dataset `_ + baseline (None or str or list or dict): `Baseline dataset `_ for the Kernel SHAP algorithm, accepted in the form of: S3 object URI, a list of rows (with at least one element), or None (for no input baseline). The baseline dataset must have the same format diff --git a/tests/unit/test_clarify.py b/tests/unit/test_clarify.py index 714f9d316c..4fe1ecb5f2 100644 --- a/tests/unit/test_clarify.py +++ b/tests/unit/test_clarify.py @@ -563,14 +563,20 @@ def test_invalid_model_predicted_label_config(): ) -def test_shap_config(): - baseline = [ - [ - 0.26124998927116394, - 0.2824999988079071, - 0.06875000149011612, - ] - ] +@pytest.mark.parametrize( + "baseline", + [ + ([[0.26124998927116394, 0.2824999988079071, 0.06875000149011612]]), + ( + { + "instances": [ + {"features": [0.26124998927116394, 0.2824999988079071, 0.06875000149011612]} + ] + } + ), + ], +) +def test_valid_shap_config(baseline): num_samples = 100 agg_method = "mean_sq" use_logit = True From 5d0fbcaefbaa8d62e3b7ba7d16f449fd99a32109 Mon Sep 17 00:00:00 2001 From: Dipanjan Kailthya Date: Sat, 29 Apr 2023 12:45:49 +0200 Subject: [PATCH 2/3] fix: Make sure LambdaStep can be created successfully when some parameters are not specified. --- req.txt | 17 +++++++++++++++++ src/sagemaker/lambda_helper.py | 13 +++---------- tests/unit/test_lambda_helper.py | 30 ++++++++++-------------------- 3 files changed, 30 insertions(+), 30 deletions(-) create mode 100644 req.txt diff --git a/req.txt b/req.txt new file mode 100644 index 0000000000..13901e97f9 --- /dev/null +++ b/req.txt @@ -0,0 +1,17 @@ +attrs>=20.3.0,<23 +boto3>=1.26.28,<2.0 +cloudpickle==2.2.1 +google-pasta +numpy>=1.9.0,<2.0 +protobuf>=3.1,<4.0 +protobuf3-to-dict>=0.1.5,<1.0 +smdebug_rulesconfig==1.0.1 +importlib-metadata>=1.4.0,<5.0 +packaging>=20.0 +pandas +pathos +schema +PyYAML==5.4.1 +jsonschema +platformdirs +tblib==1.7.0 diff --git a/src/sagemaker/lambda_helper.py b/src/sagemaker/lambda_helper.py index 568ed7eb0c..d4912d3e8a 100644 --- a/src/sagemaker/lambda_helper.py +++ b/src/sagemaker/lambda_helper.py @@ -37,7 +37,6 @@ def __init__( memory_size: int = 128, runtime: str = "python3.8", vpc_config: dict = None, - architectures: list = None, environment: dict = None, layers: list = None, ): @@ -71,8 +70,6 @@ def __init__( memory_size (int): Memory of the Lambda function in megabytes. Default is 128 MB. runtime (str): Runtime of the Lambda function. Default is set to python3.8. vpc_config (dict): VPC to deploy the Lambda function to. Default is None. - architectures (list): Which architecture to deploy to. Valid Values are - 'x86_64' and 'arm64', default is None. environment (dict): Environment Variables for the Lambda function. Default is None. layers (list): List of Lambda layers for the Lambda function. Default is None. """ @@ -87,10 +84,9 @@ def __init__( self.timeout = timeout self.memory_size = memory_size self.runtime = runtime - self.vpc_config = vpc_config - self.environment = environment - self.architectures = architectures - self.layers = layers + self.vpc_config = vpc_config or {} + self.environment = environment or {} + self.layers = layers or [] if function_arn is None and function_name is None: raise ValueError("Either function_arn or function_name must be provided.") @@ -142,7 +138,6 @@ def create(self): MemorySize=self.memory_size, VpcConfig=self.vpc_config, Environment=self.environment, - Architectures=self.architectures, Layers=self.layers, ) return response @@ -163,7 +158,6 @@ def update(self): response = lambda_client.update_function_code( FunctionName=self.function_name or self.function_arn, ZipFile=_zip_lambda_code(self.script), - Architectures=self.architectures, ) else: bucket = self.s3_bucket or self.session.default_bucket() @@ -186,7 +180,6 @@ def update(self): zipped_code_dir=self.zipped_code_dir, s3_bucket=bucket, ), - Architectures=self.architectures, ) return response except ClientError as e: diff --git a/tests/unit/test_lambda_helper.py b/tests/unit/test_lambda_helper.py index 64dc50fa68..d6ad11beb7 100644 --- a/tests/unit/test_lambda_helper.py +++ b/tests/unit/test_lambda_helper.py @@ -187,10 +187,9 @@ def test_create_lambda_happycase1(sagemaker_session): Code=code, Timeout=120, MemorySize=128, - Architectures=None, - VpcConfig=None, - Environment=None, - Layers=None, + VpcConfig={}, + Environment={}, + Layers=[], ) @@ -216,10 +215,9 @@ def test_create_lambda_happycase2(sagemaker_session): Code=code, Timeout=120, MemorySize=128, - Architectures=None, - VpcConfig=None, - Environment=None, - Layers=None, + VpcConfig={}, + Environment={}, + Layers=[], ) @@ -231,7 +229,6 @@ def test_create_lambda_happycase3(sagemaker_session): script=SCRIPT, handler=HANDLER, session=sagemaker_session, - architectures=["x86_64"], environment={"Name": "my-test-lambda"}, vpc_config={ "SubnetIds": ["test-subnet-1"], @@ -251,7 +248,6 @@ def test_create_lambda_happycase3(sagemaker_session): Code=code, Timeout=120, MemorySize=128, - Architectures=["x86_64"], VpcConfig={"SubnetIds": ["test-subnet-1"], "SecurityGroupIds": ["sec-group-1"]}, Environment={"Name": "my-test-lambda"}, Layers=["my-test-layer-1", "my-test-layer-2"], @@ -314,7 +310,6 @@ def test_update_lambda_happycase1(sagemaker_session): sagemaker_session.lambda_client.update_function_code.assert_called_with( FunctionName=FUNCTION_NAME, ZipFile=ZIPPED_CODE, - Architectures=None, ) @@ -335,7 +330,6 @@ def test_update_lambda_happycase2(sagemaker_session): FunctionName=LAMBDA_ARN, S3Bucket=S3_BUCKET, S3Key=S3_KEY, - Architectures=None, ) @@ -347,7 +341,6 @@ def test_update_lambda_happycase3(sagemaker_session): script=SCRIPT, handler=HANDLER, session=sagemaker_session, - architectures=["x86_64"], environment={"Name": "my-test-lambda"}, vpc_config={ "SubnetIds": ["test-subnet-1"], @@ -360,7 +353,6 @@ def test_update_lambda_happycase3(sagemaker_session): sagemaker_session.lambda_client.update_function_code.assert_called_with( FunctionName=FUNCTION_NAME, ZipFile=ZIPPED_CODE, - Architectures=["x86_64"], ) @@ -380,7 +372,6 @@ def test_update_lambda_s3bucket_not_provided(s3_upload, sagemaker_session): FunctionName=LAMBDA_ARN, S3Bucket=sagemaker_session.default_bucket(), S3Key=s3_upload.return_value, - Architectures=None, ) @@ -425,10 +416,9 @@ def test_upsert_lambda_happycase1(sagemaker_session): Code=code, Timeout=120, MemorySize=128, - Architectures=None, - VpcConfig=None, - Environment=None, - Layers=None, + VpcConfig={}, + Environment={}, + Layers=[], ) @@ -455,7 +445,7 @@ def test_upsert_lambda_happycase2(sagemaker_session): lambda_obj.upsert() sagemaker_session.lambda_client.update_function_code.assert_called_once_with( - FunctionName=FUNCTION_NAME, ZipFile=ZIPPED_CODE, Architectures=None + FunctionName=FUNCTION_NAME, ZipFile=ZIPPED_CODE ) From 1e6df605e80fb09e48f2c224fab4f7827233eb19 Mon Sep 17 00:00:00 2001 From: Dipanjan Kailthya Date: Sat, 29 Apr 2023 12:46:26 +0200 Subject: [PATCH 3/3] fix: remove unnecessary file --- req.txt | 17 ----------------- 1 file changed, 17 deletions(-) delete mode 100644 req.txt diff --git a/req.txt b/req.txt deleted file mode 100644 index 13901e97f9..0000000000 --- a/req.txt +++ /dev/null @@ -1,17 +0,0 @@ -attrs>=20.3.0,<23 -boto3>=1.26.28,<2.0 -cloudpickle==2.2.1 -google-pasta -numpy>=1.9.0,<2.0 -protobuf>=3.1,<4.0 -protobuf3-to-dict>=0.1.5,<1.0 -smdebug_rulesconfig==1.0.1 -importlib-metadata>=1.4.0,<5.0 -packaging>=20.0 -pandas -pathos -schema -PyYAML==5.4.1 -jsonschema -platformdirs -tblib==1.7.0