reformat

Author: atqy

ci-scripts/queue_build.py

@@ -51,10 +51,18 @@ def _wait_for_other_builds(ticket_number):
         file_ticket_number, build_id, source_version = _build_info_from_file(file)
         print(
             "%s -> %s %s, ticket number: %s status: %s"
-            % (order, build_id, source_version, file_ticket_number, file.key.split("/")[1])
+            % (
+                order,
+                build_id,
+                source_version,
+                file_ticket_number,
+                file.key.split("/")[1],
+            )
         )
     print()
-    build_id = re.sub("[_/]", "-", os.environ.get("CODEBUILD_BUILD_ID", "CODEBUILD-BUILD-ID"))
+    build_id = re.sub(
+        "[_/]", "-", os.environ.get("CODEBUILD_BUILD_ID", "CODEBUILD-BUILD-ID")
+    )
     source_version = re.sub(
         "[_/]",
         "-",
@@ -68,7 +76,9 @@ def _wait_for_other_builds(ticket_number):
         _cleanup_tickets_with_terminal_states()
         waiting_tickets = _list_tickets("waiting")
         if waiting_tickets:
-            first_waiting_ticket_number, _, _ = _build_info_from_file(_list_tickets("waiting")[0])
+            first_waiting_ticket_number, _, _ = _build_info_from_file(
+                _list_tickets("waiting")[0]
+            )
         else:
             first_waiting_ticket_number = ticket_number
 
@@ -91,7 +101,9 @@ def last_in_progress_elapsed_time_check():
     in_progress_tickets = _list_tickets("in-progress")
     if not in_progress_tickets:
         return True
-    last_in_progress_ticket, _, _ = _build_info_from_file(_list_tickets("in-progress")[-1])
+    last_in_progress_ticket, _, _ = _build_info_from_file(
+        _list_tickets("in-progress")[-1]
+    )
     _elapsed_time = int(1000 * time.time()) - last_in_progress_ticket
     last_in_progress_elapsed_time = int(_elapsed_time / (1000 * 60))  # in minutes
     return last_in_progress_elapsed_time > INTERVAL_BETWEEN_CONCURRENT_RUNS

doc/conf.py

@@ -85,7 +85,13 @@
     "https://cdn.datatables.net/1.10.23/css/jquery.dataTables.min.css",
 ]
 
-html_context = {"css_files": ["_static/theme_overrides.css", "_static/pagination.css", "_static/search_accessories.css"]}
+html_context = {
+    "css_files": [
+        "_static/theme_overrides.css",
+        "_static/pagination.css",
+        "_static/search_accessories.css",
+    ]
+}
 
 # Example configuration for intersphinx: refer to the Python standard library.
 intersphinx_mapping = {"http://docs.python.org/": None}

doc/doc_utils/jumpstart_doc_utils.py

@@ -60,8 +60,10 @@ class ProblemTypes(str, Enum):
 
 JUMPSTART_REGION = "eu-west-2"
 SDK_MANIFEST_FILE = "models_manifest.json"
-JUMPSTART_BUCKET_BASE_URL = "https://jumpstart-cache-prod-{}.s3.{}.amazonaws.com".format(
-    JUMPSTART_REGION, JUMPSTART_REGION
+JUMPSTART_BUCKET_BASE_URL = (
+    "https://jumpstart-cache-prod-{}.s3.{}.amazonaws.com".format(
+        JUMPSTART_REGION, JUMPSTART_REGION
+    )
 )
 TASK_MAP = {
     Tasks.IC: ProblemTypes.IMAGE_CLASSIFICATION,
@@ -187,7 +189,9 @@ def create_jumpstart_model_table():
         file_content.append("     - {}\n".format(model["min_version"]))
         file_content.append("     - {}\n".format(model_task))
         file_content.append(
-            "     - `{} <{}>`__ |external-link|\n".format(model_source, model_spec["url"])
+            "     - `{} <{}>`__ |external-link|\n".format(
+                model_source, model_spec["url"]
+            )
         )
 
     f = open("doc_utils/pretrainedmodels.rst", "w")

setup.py

@@ -69,15 +69,19 @@ def read_requirements(filename):
 # Meta dependency groups
 extras["all"] = [item for group in extras.values() for item in group]
 # Tests specific dependencies (do not need to be included in 'all')
-extras["test"] = (extras["all"] + read_requirements("requirements/extras/test_requirements.txt"),)
+extras["test"] = (
+    extras["all"] + read_requirements("requirements/extras/test_requirements.txt"),
+)
 
 setup(
     name="sagemaker",
     version=read_version(),
     description="Open source library for training and deploying models on Amazon SageMaker.",
     packages=find_packages("src"),
     package_dir={"": "src"},
-    py_modules=[os.path.splitext(os.path.basename(path))[0] for path in glob("src/*.py")],
+    py_modules=[
+        os.path.splitext(os.path.basename(path))[0] for path in glob("src/*.py")
+    ],
     include_package_data=True,
     long_description=read("README.rst"),
     author="Amazon Web Services",

src/sagemaker/__init__.py

@@ -28,7 +28,9 @@
     FactorizationMachines,
     FactorizationMachinesModel,
 )
-from sagemaker.amazon.factorization_machines import FactorizationMachinesPredictor  # noqa: F401
+from sagemaker.amazon.factorization_machines import (
+    FactorizationMachinesPredictor,
+)  # noqa: F401
 from sagemaker.inputs import TrainingInput  # noqa: F401
 from sagemaker.amazon.ntm import NTM, NTMModel, NTMPredictor  # noqa: F401
 from sagemaker.amazon.randomcutforest import (  # noqa: F401
@@ -45,11 +47,18 @@
 )
 
 from sagemaker.algorithm import AlgorithmEstimator  # noqa: F401
-from sagemaker.analytics import TrainingJobAnalytics, HyperparameterTuningJobAnalytics  # noqa: F401
+from sagemaker.analytics import (
+    TrainingJobAnalytics,
+    HyperparameterTuningJobAnalytics,
+)  # noqa: F401
 from sagemaker.local.local_session import LocalSession  # noqa: F401
 
 from sagemaker.model import Model, ModelPackage  # noqa: F401
-from sagemaker.model_metrics import ModelMetrics, MetricsSource, FileSource  # noqa: F401
+from sagemaker.model_metrics import (
+    ModelMetrics,
+    MetricsSource,
+    FileSource,
+)  # noqa: F401
 from sagemaker.pipeline import PipelineModel  # noqa: F401
 from sagemaker.predictor import Predictor  # noqa: F401
 from sagemaker.processing import Processor, ScriptProcessor  # noqa: F401
@@ -60,6 +69,9 @@
 from sagemaker.session import get_execution_role  # noqa: F401
 
 from sagemaker.automl.automl import AutoML, AutoMLJob, AutoMLInput  # noqa: F401
-from sagemaker.automl.candidate_estimator import CandidateEstimator, CandidateStep  # noqa: F401
+from sagemaker.automl.candidate_estimator import (
+    CandidateEstimator,
+    CandidateStep,
+)  # noqa: F401
 
 __version__ = importlib_metadata.version("sagemaker")

src/sagemaker/algorithm.py

@@ -168,8 +168,10 @@ def __init__(
             max_wait=max_wait,
         )
 
-        self.algorithm_spec = self.sagemaker_session.sagemaker_client.describe_algorithm(
-            AlgorithmName=algorithm_arn
+        self.algorithm_spec = (
+            self.sagemaker_session.sagemaker_client.describe_algorithm(
+                AlgorithmName=algorithm_arn
+            )
         )
         self.validate_train_spec()
         self.hyperparameter_definitions = self._parse_hyperparameters()
@@ -185,7 +187,9 @@ def validate_train_spec(self):
 
         # Check that the input mode provided is compatible with the training input modes for the
         # algorithm.
-        input_modes = self._algorithm_training_input_modes(train_spec["TrainingChannels"])
+        input_modes = self._algorithm_training_input_modes(
+            train_spec["TrainingChannels"]
+        )
         if self.input_mode not in input_modes:
             raise ValueError(
                 "Invalid input mode: %s. %s only supports: %s"
@@ -233,7 +237,9 @@ def training_image_uri(self):
         The fit() method, that does the model training, calls this method to
         find the image to use for model training.
         """
-        raise RuntimeError("training_image_uri is never meant to be called on Algorithm Estimators")
+        raise RuntimeError(
+            "training_image_uri is never meant to be called on Algorithm Estimators"
+        )
 
     def enable_network_isolation(self):
         """Return True if this Estimator will need network isolation to run.
@@ -377,7 +383,9 @@ def transformer(
 
             tags = tags or self.tags
         else:
-            raise RuntimeError("No finished training job found associated with this estimator")
+            raise RuntimeError(
+                "No finished training job found associated with this estimator"
+            )
 
         return Transformer(
             model_name,
@@ -431,21 +439,29 @@ def _validate_input_channels(self, channels):
         for c in channels:
             if c not in training_channels:
                 raise ValueError(
-                    "Unknown input channel: %s is not supported by: %s" % (c, algorithm_name)
+                    "Unknown input channel: %s is not supported by: %s"
+                    % (c, algorithm_name)
                 )
 
         # check for required channels that were not provided
         for name, channel in training_channels.items():
-            if name not in channels and "IsRequired" in channel and channel["IsRequired"]:
-                raise ValueError("Required input channel: %s Was not provided." % (name))
+            if (
+                name not in channels
+                and "IsRequired" in channel
+                and channel["IsRequired"]
+            ):
+                raise ValueError(
+                    "Required input channel: %s Was not provided." % (name)
+                )
 
     def _validate_and_cast_hyperparameter(self, name, v):
         """Placeholder docstring"""
         algorithm_name = self.algorithm_spec["AlgorithmName"]
 
         if name not in self.hyperparameter_definitions:
             raise ValueError(
-                "Invalid hyperparameter: %s is not supported by %s" % (name, algorithm_name)
+                "Invalid hyperparameter: %s is not supported by %s"
+                % (name, algorithm_name)
             )
 
         definition = self.hyperparameter_definitions[name]
@@ -456,7 +472,9 @@ def _validate_and_cast_hyperparameter(self, name, v):
 
         if "range" in definition and not definition["range"].is_valid(value):
             valid_range = definition["range"].as_tuning_range(name)
-            raise ValueError("Invalid value: %s Supported range: %s" % (value, valid_range))
+            raise ValueError(
+                "Invalid value: %s Supported range: %s" % (value, valid_range)
+            )
         return value
 
     def _validate_and_set_default_hyperparameters(self):
@@ -544,7 +562,9 @@ def _algorithm_training_input_modes(self, training_channels):
         return current_input_modes
 
     @classmethod
-    def _prepare_init_params_from_job_description(cls, job_details, model_channel_name=None):
+    def _prepare_init_params_from_job_description(
+        cls, job_details, model_channel_name=None
+    ):
         """Convert the job description to init params that can be handled by the class constructor.
 
         Args:
@@ -556,9 +576,9 @@ def _prepare_init_params_from_job_description(cls, job_details, model_channel_na
         Returns:
             dict: The transformed init_params
         """
-        init_params = super(AlgorithmEstimator, cls)._prepare_init_params_from_job_description(
-            job_details, model_channel_name
-        )
+        init_params = super(
+            AlgorithmEstimator, cls
+        )._prepare_init_params_from_job_description(job_details, model_channel_name)
 
         # This hyperparameter is added by Amazon SageMaker Automatic Model Tuning.
         # It cannot be set through instantiating an estimator.

src/sagemaker/amazon/amazon_estimator.py

@@ -117,14 +117,18 @@ def data_location(self, data_location):
         """Placeholder docstring"""
         if not data_location.startswith("s3://"):
             raise ValueError(
-                'Expecting an S3 URL beginning with "s3://". Got "{}"'.format(data_location)
+                'Expecting an S3 URL beginning with "s3://". Got "{}"'.format(
+                    data_location
+                )
             )
         if data_location[-1] != "/":
             data_location = data_location + "/"
         self._data_location = data_location
 
     @classmethod
-    def _prepare_init_params_from_job_description(cls, job_details, model_channel_name=None):
+    def _prepare_init_params_from_job_description(
+        cls, job_details, model_channel_name=None
+    ):
         """Convert the job description to init params that can be handled by the class constructor.
 
         Args:
@@ -152,7 +156,9 @@ def _prepare_init_params_from_job_description(cls, job_details, model_channel_na
         del init_params["image_uri"]
         return init_params
 
-    def prepare_workflow_for_training(self, records=None, mini_batch_size=None, job_name=None):
+    def prepare_workflow_for_training(
+        self, records=None, mini_batch_size=None, job_name=None
+    ):
         """Calls _prepare_for_training. Used when setting up a workflow.
 
         Args:
@@ -178,7 +184,9 @@ def _prepare_for_training(self, records, mini_batch_size=None, job_name=None):
                 specified, one is generated, using the base name given to the
                 constructor if applicable.
         """
-        super(AmazonAlgorithmEstimatorBase, self)._prepare_for_training(job_name=job_name)
+        super(AmazonAlgorithmEstimatorBase, self)._prepare_for_training(
+            job_name=job_name
+        )
 
         feature_dim = None
 
@@ -244,7 +252,9 @@ def fit(
                 will be unassociated.
                 * `TrialComponentDisplayName` is used for display in Studio.
         """
-        self._prepare_for_training(records, job_name=job_name, mini_batch_size=mini_batch_size)
+        self._prepare_for_training(
+            records, job_name=job_name, mini_batch_size=mini_batch_size
+        )
 
         self.latest_training_job = _TrainingJob.start_new(
             self, records, experiment_config=experiment_config
@@ -287,7 +297,9 @@ def record_set(self, train, labels=None, channel="train", encrypt=False):
         )
         parsed_s3_url = urlparse(self.data_location)
         bucket, key_prefix = parsed_s3_url.netloc, parsed_s3_url.path
-        key_prefix = key_prefix + "{}-{}/".format(type(self).__name__, sagemaker_timestamp())
+        key_prefix = key_prefix + "{}-{}/".format(
+            type(self).__name__, sagemaker_timestamp()
+        )
         key_prefix = key_prefix.lstrip("/")
         logger.debug("Uploading to bucket %s and key_prefix %s", bucket, key_prefix)
         manifest_s3_file = upload_numpy_to_s3_shards(
@@ -404,7 +416,10 @@ def _build_shards(num_shards, array):
     shard_size = int(array.shape[0] / num_shards)
     if shard_size == 0:
         raise ValueError("Array length is less than num shards")
-    shards = [array[i * shard_size : i * shard_size + shard_size] for i in range(num_shards - 1)]
+    shards = [
+        array[i * shard_size : i * shard_size + shard_size]
+        for i in range(num_shards - 1)
+    ]
     shards.append(array[(num_shards - 1) * shard_size :])
     return shards
 
@@ -451,7 +466,9 @@ def upload_numpy_to_s3_shards(
         manifest_str = json.dumps(
             [{"prefix": "s3://{}/{}".format(bucket, key_prefix)}] + uploaded_files
         )
-        s3.Object(bucket, manifest_key).put(Body=manifest_str.encode("utf-8"), **extra_put_kwargs)
+        s3.Object(bucket, manifest_key).put(
+            Body=manifest_str.encode("utf-8"), **extra_put_kwargs
+        )
         return "s3://{}/{}".format(bucket, manifest_key)
     except Exception as ex:  # pylint: disable=broad-except
         try:

src/sagemaker/amazon/common.py

@@ -53,7 +53,8 @@ def serialize(self, data):
 
         if len(data.shape) != 2:
             raise ValueError(
-                "Expected a 1D or 2D array, but got a %dD array instead." % len(data.shape)
+                "Expected a 1D or 2D array, but got a %dD array instead."
+                % len(data.shape)
             )
 
         buffer = io.BytesIO()
@@ -290,5 +291,7 @@ def _resolve_type(dtype):
     raise ValueError("Unsupported dtype {} on array".format(dtype))
 
 
-numpy_to_record_serializer = deprecated_class(RecordSerializer, "numpy_to_record_serializer")
+numpy_to_record_serializer = deprecated_class(
+    RecordSerializer, "numpy_to_record_serializer"
+)
 record_deserializer = deprecated_class(RecordDeserializer, "record_deserializer")

src/sagemaker/amazon/factorization_machines.py

@@ -222,7 +222,9 @@ def __init__(
             :class:`~sagemaker.estimator.amazon_estimator.AmazonAlgorithmEstimatorBase` and
             :class:`~sagemaker.estimator.EstimatorBase`.
         """
-        super(FactorizationMachines, self).__init__(role, instance_count, instance_type, **kwargs)
+        super(FactorizationMachines, self).__init__(
+            role, instance_count, instance_type, **kwargs
+        )
 
         self.num_factors = num_factors
         self.predictor_type = predictor_type
@@ -353,7 +355,9 @@ def __init__(
             sagemaker_session.boto_region_name,
             version=FactorizationMachines.repo_version,
         )
-        pop_out_unused_kwarg("predictor_cls", kwargs, FactorizationMachinesPredictor.__name__)
+        pop_out_unused_kwarg(
+            "predictor_cls", kwargs, FactorizationMachinesPredictor.__name__
+        )
         pop_out_unused_kwarg("image_uri", kwargs, image_uri)
         super(FactorizationMachinesModel, self).__init__(
             image_uri,