Merge branch 'master' into feat/jumpstart-instance-types

Author: evakravi

.coveragerc

@@ -2,3 +2,4 @@
 concurrency = threading
 omit = sagemaker/tests/*
 timid = True
+disable_warnings = module-not-measured

CHANGELOG.md

@@ -1,5 +1,16 @@
 # Changelog
 
+## v2.140.1 (2023-03-21)
+
+### Bug Fixes and Other Changes
+
+ * Fix cross account register model
+ * Handle instance support for Hugging Face tests
+ * Upgrade apache-airflow-providers-amazon version
+ * build(deps): bump apache-airflow from 2.4.1 to 2.5.1
+ * Mark test_create_model_package test for xfail
+ * Disable module-not-measured warnings to avoid clutter in build logs
+
 ## v2.140.0 (2023-03-17)
 
 ### Features

VERSION

@@ -1 +1 @@
-2.140.1.dev0
+2.140.2.dev0

requirements/extras/test_requirements.txt

@@ -12,8 +12,8 @@ awslogs==0.14.0
 black==22.3.0
 stopit==1.1.2
 # Update tox.ini to have correct version of airflow constraints file
-apache-airflow==2.4.1
-apache-airflow-providers-amazon==4.0.0
+apache-airflow==2.5.1
+apache-airflow-providers-amazon==7.2.1
 attrs==22.1.0
 fabric==2.6.0
 requests==2.27.1

src/sagemaker/session.py

@@ -3222,7 +3222,9 @@ def create_model_package_from_containers(
         )
 
         def submit(request):
-            if model_package_group_name is not None:
+            if model_package_group_name is not None and not model_package_group_name.startswith(
+                "arn:"
+            ):
                 _create_resource(
                     lambda: self.sagemaker_client.create_model_package_group(
                         ModelPackageGroupName=request["ModelPackageGroupName"]

tests/conftest.py

@@ -567,11 +567,16 @@ def gpu_instance_type(sagemaker_session, request):
 
 @pytest.fixture()
 def gpu_pytorch_instance_type(sagemaker_session, request):
-    if "pytorch_inference_version" in request.fixturenames:
-        fw_version = request.getfixturevalue("pytorch_inference_version")
-    else:
+    fw_version = None
+    for pytorch_version_fixture in [
+        "pytorch_inference_version",
+        "huggingface_training_pytorch_latest_version",
+        "huggingface_inference_pytorch_latest_version",
+    ]:
+        if pytorch_version_fixture in request.fixturenames:
+            fw_version = request.getfixturevalue(pytorch_version_fixture)
+    if fw_version is None:
         fw_version = request.param
-
     region = sagemaker_session.boto_session.region_name
     if region in NO_P3_REGIONS:
         if Version(fw_version) >= Version("1.13"):

tests/integ/test_airflow_config.py

@@ -56,8 +56,8 @@
         import sagemaker.workflow.airflow as sm_airflow
         import airflow.utils as utils
         from airflow import DAG
-        from airflow.providers.amazon.aws.operators.sagemaker import SageMakerTrainingOperator
-        from airflow.providers.amazon.aws.operators.sagemaker_transform import (
+        from airflow.providers.amazon.aws.operators.sagemaker import (
+            SageMakerTrainingOperator,
             SageMakerTransformOperator,
         )
 

tests/integ/test_huggingface.py

@@ -19,27 +19,19 @@
 from sagemaker.huggingface import HuggingFace, HuggingFaceProcessor
 from sagemaker.huggingface.model import HuggingFaceModel, HuggingFacePredictor
 from sagemaker.utils import unique_name_from_base
-from tests import integ
-from tests.integ.utils import gpu_list, retry_with_instance_list
 from tests.integ import DATA_DIR, TRAINING_DEFAULT_TIMEOUT_MINUTES
 from tests.integ.timeout import timeout, timeout_and_delete_endpoint_by_name
 
 ROLE = "SageMakerRole"
 
 
 @pytest.mark.release
-@pytest.mark.skipif(
-    integ.test_region() in integ.TRAINING_NO_P2_REGIONS
-    and integ.test_region() in integ.TRAINING_NO_P3_REGIONS,
-    reason="no ml.p2 or ml.p3 instances in this region",
-)
-@retry_with_instance_list(gpu_list(integ.test_region()))
 def test_framework_processing_job_with_deps(
     sagemaker_session,
     huggingface_training_latest_version,
     huggingface_training_pytorch_latest_version,
     huggingface_pytorch_latest_training_py_version,
-    **kwargs,
+    gpu_pytorch_instance_type,
 ):
     with timeout(minutes=TRAINING_DEFAULT_TIMEOUT_MINUTES):
         code_path = os.path.join(DATA_DIR, "dummy_code_bundle_with_reqs")
@@ -51,7 +43,7 @@ def test_framework_processing_job_with_deps(
             py_version=huggingface_pytorch_latest_training_py_version,
             role=ROLE,
             instance_count=1,
-            instance_type=kwargs["instance_type"],
+            instance_type=gpu_pytorch_instance_type,
             sagemaker_session=sagemaker_session,
             base_job_name="test-huggingface",
         )
@@ -64,18 +56,12 @@ def test_framework_processing_job_with_deps(
 
 
 @pytest.mark.release
-@pytest.mark.skipif(
-    integ.test_region() in integ.TRAINING_NO_P2_REGIONS
-    and integ.test_region() in integ.TRAINING_NO_P3_REGIONS,
-    reason="no ml.p2 or ml.p3 instances in this region",
-)
-@retry_with_instance_list(gpu_list(integ.test_region()))
 def test_huggingface_training(
     sagemaker_session,
     huggingface_training_latest_version,
     huggingface_training_pytorch_latest_version,
     huggingface_pytorch_latest_training_py_version,
-    **kwargs,
+    gpu_pytorch_instance_type,
 ):
     with timeout(minutes=TRAINING_DEFAULT_TIMEOUT_MINUTES):
         data_path = os.path.join(DATA_DIR, "huggingface")
@@ -87,7 +73,7 @@ def test_huggingface_training(
             transformers_version=huggingface_training_latest_version,
             pytorch_version=huggingface_training_pytorch_latest_version,
             instance_count=1,
-            instance_type=kwargs["instance_type"],
+            instance_type=gpu_pytorch_instance_type,
             hyperparameters={
                 "model_name_or_path": "distilbert-base-cased",
                 "task_name": "wnli",
@@ -111,14 +97,6 @@ def test_huggingface_training(
 
 
 @pytest.mark.release
-@pytest.mark.skipif(
-    integ.test_region() in integ.TRAINING_NO_P2_REGIONS
-    and integ.test_region() in integ.TRAINING_NO_P3_REGIONS,
-    reason="no ml.p2 or ml.p3 instances in this region",
-)
-@pytest.mark.skip(
-    reason="need to re enable it later t.corp:V609860141",
-)
 def test_huggingface_training_tf(
     sagemaker_session,
     gpu_instance_type,
@@ -161,7 +139,7 @@ def test_huggingface_training_tf(
 )
 def test_huggingface_inference(
     sagemaker_session,
-    gpu_instance_type,
+    gpu_pytorch_instance_type,
     huggingface_inference_latest_version,
     huggingface_inference_pytorch_latest_version,
     huggingface_pytorch_latest_inference_py_version,
@@ -182,7 +160,9 @@ def test_huggingface_inference(
     )
     with timeout_and_delete_endpoint_by_name(endpoint_name, sagemaker_session):
         model.deploy(
-            instance_type=gpu_instance_type, initial_instance_count=1, endpoint_name=endpoint_name
+            instance_type=gpu_pytorch_instance_type,
+            initial_instance_count=1,
+            endpoint_name=endpoint_name,
         )
 
         predictor = HuggingFacePredictor(endpoint_name=endpoint_name)

tests/integ/test_marketplace.py

@@ -233,6 +233,7 @@ def iris_image(sagemaker_session):
     _delete_repository(ecr_client, algorithm_name)
 
 
+@pytest.mark.xfail(reason="marking this for xfail until we work on the test failure to be fixed")
 def test_create_model_package(sagemaker_session, boto_session, iris_image):
     MODEL_NAME = "iris-classifier-mp"
     # Prepare

tests/unit/test_session.py

@@ -2650,6 +2650,35 @@ def test_create_model_package_from_containers(sagemaker_session):
     sagemaker_session.sagemaker_client.create_model_package.assert_called_once()
 
 
+def test_create_model_package_from_containers_cross_account_mpg_name(sagemaker_session):
+    mpg_name = "arn:aws:sagemaker:us-east-1:215995503607:model-package-group/stage-dev"
+    content_types = ["text/csv"]
+    response_types = ["text/csv"]
+    sagemaker_session.create_model_package_from_containers(
+        model_package_group_name=mpg_name,
+        content_types=content_types,
+        response_types=response_types,
+    )
+    sagemaker_session.sagemaker_client.create_model_package.assert_called_once()
+
+
+def test_create_mpg_from_containers_cross_account_mpg_name(sagemaker_session):
+    mpg_name = "arn:aws:sagemaker:us-east-1:215995503607:model-package-group/stage-dev"
+    content_types = ["text/csv"]
+    response_types = ["text/csv"]
+    with pytest.raises(AssertionError) as error:
+        sagemaker_session.create_model_package_from_containers(
+            model_package_group_name=mpg_name,
+            content_types=content_types,
+            response_types=response_types,
+        )
+        sagemaker_session.sagemaker_client.create_model_package_group.assert_called_once()
+        assert (
+            "Expected 'create_model_package_group' to have been called once. "
+            "Called 0 times." == str(error)
+        )
+
+
 def test_create_model_package_from_containers_name_conflict(sagemaker_session):
     model_package_name = "sagemaker-model-package"
     model_package_group_name = "sagemaker-model-package-group"

tox.ini

@@ -73,7 +73,7 @@ passenv =
 # Can be used to specify which tests to run, e.g.: tox -- -s
 commands =
     python -c "import os; os.system('install-custom-pkgs --install-boto-wheels')"
-    pip install 'apache-airflow==2.4.1' --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.4.1/constraints-3.10.txt"
+    pip install 'apache-airflow==2.5.1' --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.5.1/constraints-3.7.txt"
 
     pytest --cov=sagemaker --cov-append {posargs}
     {env:IGNORE_COVERAGE:} coverage report -i --fail-under=86