MultiPartCopy with Sync Algorithm (aws#4475)

* first pass at sync function with util classes

* adding tests and update clases

* linting

* file generator class inheritance

* lint

* multipart copy and algorithm updates

* modularize sync

* reformatting folders

* testing for sync

* do not tolerate vulnerable

* remove prints

* handle multithreading progress bar

* update tests

* optimize function and add hub bucket prefix

* docstrings and linting

Author: bencrabtree

src/sagemaker/jumpstart/cache.py

@@ -474,7 +474,9 @@ def _retrieval_function(
                 formatted_content=hub_description,
             )
 
-        raise ValueError(self._file_type_error_msg(data_type))
+        raise ValueError(
+            self._file_type_error_msg(data_type)
+        )
 
     def get_manifest(
         self,

tests/unit/sagemaker/jumpstart/curated_hub/test_utils.py

@@ -147,10 +147,7 @@ def test_generate_hub_arn_for_init_kwargs():
         utils.generate_hub_arn_for_init_kwargs(hub_arn, "us-east-1", mock_custom_session) == hub_arn
     )
 
-    assert (
-        utils.generate_hub_arn_for_estimator_init_kwargs(hub_arn, None, mock_custom_session)
-        == hub_arn
-    )
+    assert utils.generate_hub_arn_for_init_kwargs(hub_arn, None, mock_custom_session) == hub_arn
 
 
 def test_generate_default_hub_bucket_name():
@@ -170,8 +167,14 @@ def test_create_hub_bucket_if_it_does_not_exist():
     mock_sagemaker_session.client("sts").get_caller_identity.return_value = {
         "Account": "123456789123"
     }
+    hub_arn = "arn:aws:sagemaker:us-west-2:12346789123:hub/my-awesome-hub"
+    # Mock custom session with custom values
+    mock_custom_session = Mock()
+    mock_custom_session.account_id.return_value = "000000000000"
+    mock_custom_session.boto_region_name = "us-east-2"
     mock_sagemaker_session.boto_session.resource("s3").Bucket().creation_date = None
     mock_sagemaker_session.boto_region_name = "us-east-1"
+
     bucket_name = "sagemaker-hubs-us-east-1-123456789123"
     created_hub_bucket_name = utils.create_hub_bucket_if_it_does_not_exist(
         sagemaker_session=mock_sagemaker_session

tests/unit/sagemaker/jumpstart/test_cache.py

@@ -1134,7 +1134,7 @@ def test_jumpstart_local_metadata_override_specs_not_exist_both_directories(
 
     mocked_is_dir.assert_any_call("/some/directory/metadata/manifest/root")
     assert mocked_is_dir.call_count == 2
-    assert mocked_open.call_count == 2
+    mocked_open.assert_not_called()
     mocked_get_json_file_and_etag_from_s3.assert_has_calls(
         calls=[
             call("models_manifest.json"),