diff --git a/README.rst b/README.rst index b9409f1450..787dc324ce 100644 --- a/README.rst +++ b/README.rst @@ -214,7 +214,7 @@ In order to host a SparkML model in SageMaker, it should be serialized with ``ML For more information on MLeap, see https://github.com/combust/mleap . -Supported major version of Spark: 2.4 (MLeap version - 0.9.6) +Supported major version of Spark: 3.3 (MLeap version - 0.20.0) Here is an example on how to create an instance of ``SparkMLModel`` class and use ``deploy()`` method to create an endpoint which can be used to perform prediction against your trained SparkML Model. diff --git a/src/sagemaker/image_uri_config/sparkml-serving.json b/src/sagemaker/image_uri_config/sparkml-serving.json index 3cfeeaa6bf..4fc7352710 100644 --- a/src/sagemaker/image_uri_config/sparkml-serving.json +++ b/src/sagemaker/image_uri_config/sparkml-serving.json @@ -58,6 +58,37 @@ "us-west-2": "246618743249" }, "repository": "sagemaker-sparkml-serving" + }, + "3.3": { + "registries": { + "af-south-1": "510948584623", + "ap-east-1": "651117190479", + "ap-northeast-1": "354813040037", + "ap-northeast-2": "366743142698", + "ap-northeast-3": "867004704886", + "ap-south-1": "720646828776", + "ap-southeast-1": "121021644041", + "ap-southeast-2": "783357654285", + "ap-southeast-3": "951798379941", + "ca-central-1": "341280168497", + "cn-north-1": "450853457545", + "cn-northwest-1": "451049120500", + "eu-central-1": "492215442770", + "eu-north-1": "662702820516", + "eu-west-1": "141502667606", + "eu-west-2": "764974769150", + "eu-west-3": "659782779980", + "eu-south-1": "978288397137", + "me-south-1": "801668240914", + "sa-east-1": "737474898029", + "us-east-1": "683313688378", + "us-east-2": "257758044811", + "us-gov-west-1": "414596584902", + "us-iso-east-1": "833128469047", + "us-west-1": "746614075791", + "us-west-2": "246618743249" + }, + "repository": "sagemaker-sparkml-serving" } } } diff --git a/src/sagemaker/sparkml/model.py b/src/sagemaker/sparkml/model.py index 2fdea591d6..cdbeeec249 100644 --- a/src/sagemaker/sparkml/model.py +++ b/src/sagemaker/sparkml/model.py @@ -78,7 +78,7 @@ def __init__( self, model_data: Union[str, PipelineVariable], role: Optional[str] = None, - spark_version: str = "2.4", + spark_version: str = "3.3", sagemaker_session: Optional[Session] = None, **kwargs, ): @@ -95,7 +95,7 @@ def __init__( artifacts. After the endpoint is created, the inference code might use the IAM role, if it needs to access an AWS resource. spark_version (str): Spark version you want to use for executing the - inference (default: '2.4'). + inference (default: '3.3'). sagemaker_session (sagemaker.session.Session): Session object which manages interactions with Amazon SageMaker APIs and any other AWS services needed. If not specified, the estimator creates one diff --git a/tests/unit/sagemaker/image_uris/test_sparkml.py b/tests/unit/sagemaker/image_uris/test_sparkml.py index 3addf34d33..2f3298861a 100644 --- a/tests/unit/sagemaker/image_uris/test_sparkml.py +++ b/tests/unit/sagemaker/image_uris/test_sparkml.py @@ -43,7 +43,7 @@ "us-west-1": "746614075791", "us-west-2": "246618743249", } -VERSIONS = ["2.2", "2.4"] +VERSIONS = ["2.2", "2.4", "3.3"] @pytest.mark.parametrize("version", VERSIONS) diff --git a/tests/unit/sagemaker/workflow/test_steps.py b/tests/unit/sagemaker/workflow/test_steps.py index 6161537220..9887d43078 100644 --- a/tests/unit/sagemaker/workflow/test_steps.py +++ b/tests/unit/sagemaker/workflow/test_steps.py @@ -918,7 +918,7 @@ def test_create_model_step_with_model_pipeline(tfo, time, sagemaker_session): }, { "Environment": {"SAGEMAKER_DEFAULT_INVOCATIONS_ACCEPT": "text/csv"}, - "Image": "246618743249.dkr.ecr.us-west-2.amazonaws.com/sagemaker-sparkml-serving:2.4", + "Image": "246618743249.dkr.ecr.us-west-2.amazonaws.com/sagemaker-sparkml-serving:3.3", "ModelDataUrl": "s3://bucket/model_2.tar.gz", }, ], diff --git a/tests/unit/test_pipeline_model.py b/tests/unit/test_pipeline_model.py index f4fb892d21..38c9b373e3 100644 --- a/tests/unit/test_pipeline_model.py +++ b/tests/unit/test_pipeline_model.py @@ -97,7 +97,7 @@ def test_prepare_container_def(tfo, time, sagemaker_session): { "Environment": {"SAGEMAKER_DEFAULT_INVOCATIONS_ACCEPT": "text/csv"}, "Image": "246618743249.dkr.ecr.us-west-2.amazonaws.com" - + "/sagemaker-sparkml-serving:2.4", + + "/sagemaker-sparkml-serving:3.3", "ModelDataUrl": "s3://bucket/model_2.tar.gz", }, ] @@ -338,7 +338,7 @@ def test_network_isolation(tfo, time, sagemaker_session): "ModelDataUrl": "s3://bucket/model_1.tar.gz", }, { - "Image": "246618743249.dkr.ecr.us-west-2.amazonaws.com/sagemaker-sparkml-serving:2.4", + "Image": "246618743249.dkr.ecr.us-west-2.amazonaws.com/sagemaker-sparkml-serving:3.3", "Environment": {}, "ModelDataUrl": "s3://bucket/model_2.tar.gz", }, diff --git a/tests/unit/test_sparkml_serving.py b/tests/unit/test_sparkml_serving.py index 660f0e1a32..ae975b1ac2 100644 --- a/tests/unit/test_sparkml_serving.py +++ b/tests/unit/test_sparkml_serving.py @@ -49,7 +49,7 @@ def sagemaker_session(): def test_sparkml_model(sagemaker_session): sparkml = SparkMLModel(sagemaker_session=sagemaker_session, model_data=MODEL_DATA, role=ROLE) - assert sparkml.image_uri == image_uris.retrieve("sparkml-serving", REGION, version="2.4") + assert sparkml.image_uri == image_uris.retrieve("sparkml-serving", REGION, version="3.3") def test_predictor_type(sagemaker_session):