Merge branch 'master' into user-short-lived-credentials-override

Author: wcarpenter1-godaddy

CHANGELOG.md

@@ -1,5 +1,12 @@
 # Changelog
 
+## v2.121.1 (2022-12-09)
+
+### Bug Fixes and Other Changes
+
+ * Pop out ModelPackageName from pipeline definition
+ * Fix failing jumpstart cache unit tests
+
 ## v2.121.0 (2022-12-08)
 
 ### Features

VERSION

@@ -1 +1 @@
-2.121.1.dev0
+2.121.2.dev0

src/sagemaker/image_uri_config/tensorflow.json

@@ -303,7 +303,8 @@
             "2.7": "2.7.0",
             "2.8": "2.8.0",
             "2.9": "2.9.2",
-            "2.10": "2.10.0"
+            "2.10": "2.10.0",
+            "2.11": "2.11.0"
         },
         "versions": {
             "1.10.0": {
@@ -1611,15 +1612,18 @@
                     "ap-northeast-2": "763104351884",
                     "ap-northeast-3": "364406365360",
                     "ap-south-1": "763104351884",
+                    "ap-south-2": "772153158452",
                     "ap-southeast-1": "763104351884",
                     "ap-southeast-2": "763104351884",
                     "ap-southeast-3": "907027046896",
                     "ca-central-1": "763104351884",
                     "cn-north-1": "727897471807",
                     "cn-northwest-1": "727897471807",
                     "eu-central-1": "763104351884",
+                    "eu-central-2": "380420809688",
                     "eu-north-1": "763104351884",
                     "eu-south-1": "692866216735",
+                    "eu-south-2": "503227376785",
                     "eu-west-1": "763104351884",
                     "eu-west-2": "763104351884",
                     "eu-west-3": "763104351884",
@@ -1642,15 +1646,52 @@
                     "ap-northeast-2": "763104351884",
                     "ap-northeast-3": "364406365360",
                     "ap-south-1": "763104351884",
+                    "ap-south-2": "772153158452",
+                    "ap-southeast-1": "763104351884",
+                    "ap-southeast-2": "763104351884",
+                    "ap-southeast-3": "907027046896",
+                    "ca-central-1": "763104351884",
+                    "cn-north-1": "727897471807",
+                    "cn-northwest-1": "727897471807",
+                    "eu-central-1": "763104351884",
+                    "eu-central-2": "380420809688",
+                    "eu-north-1": "763104351884",
+                    "eu-south-1": "692866216735",
+                    "eu-south-2": "503227376785",
+                    "eu-west-1": "763104351884",
+                    "eu-west-2": "763104351884",
+                    "eu-west-3": "763104351884",
+                    "me-south-1": "217643126080",
+                    "sa-east-1": "763104351884",
+                    "us-east-1": "763104351884",
+                    "us-east-2": "763104351884",
+                    "us-gov-west-1": "442386744353",
+                    "us-iso-east-1": "886529160074",
+                    "us-west-1": "763104351884",
+                    "us-west-2": "763104351884"
+                },
+                "repository": "tensorflow-inference"
+            },
+            "2.11.0": {
+                "registries": {
+                    "af-south-1": "626614931356",
+                    "ap-east-1": "871362719292",
+                    "ap-northeast-1": "763104351884",
+                    "ap-northeast-2": "763104351884",
+                    "ap-northeast-3": "364406365360",
+                    "ap-south-1": "763104351884",
+                    "ap-south-2": "772153158452",
                     "ap-southeast-1": "763104351884",
                     "ap-southeast-2": "763104351884",
                     "ap-southeast-3": "907027046896",
                     "ca-central-1": "763104351884",
                     "cn-north-1": "727897471807",
                     "cn-northwest-1": "727897471807",
                     "eu-central-1": "763104351884",
+                    "eu-central-2": "380420809688",
                     "eu-north-1": "763104351884",
                     "eu-south-1": "692866216735",
+                    "eu-south-2": "503227376785",
                     "eu-west-1": "763104351884",
                     "eu-west-2": "763104351884",
                     "eu-west-3": "763104351884",

src/sagemaker/workflow/_utils.py

@@ -13,6 +13,7 @@
 """Scrapper utilities to support repacking of models."""
 from __future__ import absolute_import
 
+import logging
 import os
 import shutil
 import tarfile
@@ -37,6 +38,8 @@
 if TYPE_CHECKING:
     from sagemaker.workflow.step_collections import StepCollection
 
+logger = logging.getLogger(__name__)
+
 FRAMEWORK_VERSION = "0.23-1"
 INSTANCE_TYPE = "ml.m5.large"
 REPACK_SCRIPT = "_repack_model.py"
@@ -479,10 +482,19 @@ def arguments(self) -> RequestType:
 
             request_dict = get_create_model_package_request(**model_package_args)
         # these are not available in the workflow service and will cause rejection
+        warn_msg_template = (
+            "Popping out '%s' from the pipeline definition "
+            "since it will be overridden in pipeline execution time."
+        )
         if "CertifyForMarketplace" in request_dict:
             request_dict.pop("CertifyForMarketplace")
+            logger.warning(warn_msg_template, "CertifyForMarketplace")
         if "Description" in request_dict:
             request_dict.pop("Description")
+            logger.warning(warn_msg_template, "Description")
+        if "ModelPackageName" in request_dict:
+            request_dict.pop("ModelPackageName")
+            logger.warning(warn_msg_template, "ModelPackageName")
 
         return request_dict
 

tests/integ/sagemaker/workflow/test_model_steps.py

@@ -112,6 +112,7 @@ def test_pytorch_training_model_registration_and_creation_without_custom_inferen
         inference_instances=["ml.m5.xlarge"],
         transform_instances=["ml.m5.xlarge"],
         description="test-description",
+        model_package_name="model-pkg-name-will-be-popped-out",
     )
     step_model_regis = ModelStep(
         name="pytorch-register-model",

tests/integ/test_inference_pipeline.py

@@ -50,6 +50,7 @@
 )
 
 
+@pytest.mark.skip(reason="Test has likely been failing for a while. Suspected bad XGB model.")
 def test_inference_pipeline_batch_transform(sagemaker_session, cpu_instance_type):
     sparkml_model_data = sagemaker_session.upload_data(
         path=os.path.join(SPARKML_DATA_PATH, "mleap_model.tar.gz"),

tests/unit/sagemaker/workflow/conftest.py

@@ -0,0 +1,75 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+from __future__ import absolute_import
+
+from unittest.mock import Mock, PropertyMock
+
+import pytest
+
+from sagemaker import Session
+from sagemaker.workflow.pipeline_context import PipelineSession
+
+REGION = "us-west-2"
+BUCKET = "my-bucket"
+ROLE = "DummyRole"
+IMAGE_URI = "fakeimage"
+
+
+@pytest.fixture(scope="module")
+def client():
+    """Mock client.
+
+    Considerations when appropriate:
+
+        * utilize botocore.stub.Stubber
+        * separate runtime client from client
+    """
+    client_mock = Mock()
+    client_mock._client_config.user_agent = (
+        "Boto3/1.14.24 Python/3.8.5 Linux/5.4.0-42-generic Botocore/1.17.24 Resource"
+    )
+    return client_mock
+
+
+@pytest.fixture(scope="module")
+def boto_session(client):
+    role_mock = Mock()
+    type(role_mock).arn = PropertyMock(return_value=ROLE)
+
+    resource_mock = Mock()
+    resource_mock.Role.return_value = role_mock
+
+    session_mock = Mock(region_name=REGION)
+    session_mock.resource.return_value = resource_mock
+    session_mock.client.return_value = client
+
+    return session_mock
+
+
+@pytest.fixture(scope="module")
+def pipeline_session(boto_session, client):
+    return PipelineSession(
+        boto_session=boto_session,
+        sagemaker_client=client,
+        default_bucket=BUCKET,
+    )
+
+
+@pytest.fixture(scope="module")
+def sagemaker_session(boto_session, client):
+    return Session(
+        boto_session=boto_session,
+        sagemaker_client=client,
+        sagemaker_runtime_client=client,
+        default_bucket=BUCKET,
+    )