From 5dcbb9d1c5355e052f174e13525eb973fef3b777 Mon Sep 17 00:00:00 2001 From: Lauren Yu <6631887+laurenyu@users.noreply.github.com> Date: Thu, 2 Jul 2020 16:21:30 -0700 Subject: [PATCH] breaking: rename image_name to image_uri --- src/sagemaker/chainer/estimator.py | 22 ++-- src/sagemaker/estimator.py | 28 ++--- src/sagemaker/fw_utils.py | 18 +-- src/sagemaker/mxnet/estimator.py | 26 ++-- src/sagemaker/pytorch/estimator.py | 22 ++-- src/sagemaker/rl/estimator.py | 42 +++---- src/sagemaker/sklearn/estimator.py | 26 ++-- src/sagemaker/tensorflow/estimator.py | 32 ++--- src/sagemaker/tensorflow/model.py | 2 +- src/sagemaker/transformer.py | 8 +- src/sagemaker/tuner.py | 4 +- src/sagemaker/xgboost/README.rst | 2 +- src/sagemaker/xgboost/estimator.py | 18 +-- tests/integ/test_airflow_config.py | 4 +- tests/integ/test_byo_estimator.py | 10 +- tests/integ/test_multidatamodel.py | 2 +- tests/integ/test_tuner.py | 4 +- tests/integ/test_tuner_multi_algo.py | 4 +- .../sagemaker/tensorflow/test_estimator.py | 4 +- .../tensorflow/test_estimator_attach.py | 2 +- .../tensorflow/test_estimator_init.py | 6 +- tests/unit/sagemaker/tensorflow/test_tfs.py | 2 +- tests/unit/test_airflow.py | 4 +- tests/unit/test_chainer.py | 12 +- tests/unit/test_estimator.py | 112 +++++++++--------- tests/unit/test_fw_utils.py | 52 ++++---- tests/unit/test_mxnet.py | 12 +- tests/unit/test_pytorch.py | 12 +- tests/unit/test_rl.py | 14 +-- tests/unit/test_sklearn.py | 10 +- tests/unit/test_transformer.py | 32 ++--- tests/unit/test_xgboost.py | 8 +- 32 files changed, 276 insertions(+), 280 deletions(-) diff --git a/src/sagemaker/chainer/estimator.py b/src/sagemaker/chainer/estimator.py index 674a499897..4625a3e750 100644 --- a/src/sagemaker/chainer/estimator.py +++ b/src/sagemaker/chainer/estimator.py @@ -51,7 +51,7 @@ def __init__( hyperparameters=None, framework_version=None, py_version=None, - image_name=None, + image_uri=None, **kwargs ): """This ``Estimator`` executes an Chainer script in a managed Chainer @@ -101,13 +101,13 @@ def __init__( and values, but ``str()`` will be called to convert them before training. py_version (str): Python version you want to use for executing your - model training code. Defaults to ``None``. Required unless ``image_name`` + model training code. Defaults to ``None``. Required unless ``image_uri`` is provided. framework_version (str): Chainer version you want to use for executing your model training code. Defaults to ``None``. Required unless - ``image_name`` is provided. List of supported versions: + ``image_uri`` is provided. List of supported versions: https://github.com/aws/sagemaker-python-sdk#chainer-sagemaker-estimators. - image_name (str): If specified, the estimator will use this image + image_uri (str): If specified, the estimator will use this image for training and hosting, instead of selecting the appropriate SageMaker official image based on framework_version and py_version. It can be an ECR url or dockerhub image and tag. @@ -117,7 +117,7 @@ def __init__( * ``custom-image:latest`` If ``framework_version`` or ``py_version`` are ``None``, then - ``image_name`` is required. If also ``None``, then a ``ValueError`` + ``image_uri`` is required. If also ``None``, then a ``ValueError`` will be raised. **kwargs: Additional kwargs passed to the :class:`~sagemaker.estimator.Framework` constructor. @@ -128,7 +128,7 @@ def __init__( :class:`~sagemaker.estimator.Framework` and :class:`~sagemaker.estimator.EstimatorBase`. """ - validate_version_or_image_args(framework_version, py_version, image_name) + validate_version_or_image_args(framework_version, py_version, image_uri) if py_version == "py2": logger.warning( python_deprecation_warning(self.__framework_name__, defaults.LATEST_PY2_VERSION) @@ -137,7 +137,7 @@ def __init__( self.py_version = py_version super(Chainer, self).__init__( - entry_point, source_dir, hyperparameters, image_name=image_name, **kwargs + entry_point, source_dir, hyperparameters, image_uri=image_uri, **kwargs ) self.use_mpi = use_mpi @@ -209,7 +209,7 @@ def create_model( kwargs["name"] = self._get_or_create_name(kwargs.get("name")) if "image" not in kwargs: - kwargs["image"] = self.image_name + kwargs["image"] = self.image_uri return ChainerModel( self.model_data, @@ -257,8 +257,8 @@ class constructor if value: init_params[argument[len("sagemaker_") :]] = value - image_name = init_params.pop("image") - framework, py_version, tag, _ = framework_name_from_image(image_name) + image_uri = init_params.pop("image") + framework, py_version, tag, _ = framework_name_from_image(image_uri) if tag is None: framework_version = None @@ -270,7 +270,7 @@ class constructor if not framework: # If we were unable to parse the framework name from the image it is not one of our # officially supported images, in this case just add the image to the init params. - init_params["image_name"] = image_name + init_params["image_uri"] = image_uri return init_params if framework != cls.__framework_name__: diff --git a/src/sagemaker/estimator.py b/src/sagemaker/estimator.py index 6d24a39d4c..2e7b66c85b 100644 --- a/src/sagemaker/estimator.py +++ b/src/sagemaker/estimator.py @@ -39,7 +39,6 @@ UploadedCode, validate_source_dir, _region_supports_debugger, - parameter_v2_rename_warning, ) from sagemaker.job import _Job from sagemaker.local import LocalSession @@ -1131,7 +1130,7 @@ class Estimator(EstimatorBase): def __init__( self, - image_name, + image_uri, role, train_instance_count, train_instance_type, @@ -1164,7 +1163,7 @@ def __init__( """Initialize an ``Estimator`` instance. Args: - image_name (str): The container image to use for training. + image_uri (str): The container image to use for training. role (str): An AWS IAM role (either name or full ARN). The Amazon SageMaker training jobs and APIs that create Amazon SageMaker endpoints use this role to access training data and model @@ -1273,8 +1272,7 @@ def __init__( https://docs.aws.amazon.com/sagemaker/latest/dg/API_AlgorithmSpecification.html#SageMaker-Type-AlgorithmSpecification-EnableSageMakerMetricsTimeSeries (default: ``None``). """ - logging.warning(parameter_v2_rename_warning("image_name", "image_uri")) - self.image_name = image_name + self.image_uri = image_uri self.hyperparam_dict = hyperparameters.copy() if hyperparameters else {} super(Estimator, self).__init__( role, @@ -1312,7 +1310,7 @@ def train_image(self): The fit() method, that does the model training, calls this method to find the image to use for model training. """ - return self.image_name + return self.image_uri def set_hyperparameters(self, **kwargs): """ @@ -1422,7 +1420,7 @@ class constructor job_details, model_channel_name ) - init_params["image_name"] = init_params.pop("image") + init_params["image_uri"] = init_params.pop("image") return init_params @@ -1449,7 +1447,7 @@ def __init__( enable_cloudwatch_metrics=False, container_log_level=logging.INFO, code_location=None, - image_name=None, + image_uri=None, dependencies=None, enable_network_isolation=False, git_config=None, @@ -1515,7 +1513,7 @@ def __init__( a string prepended with a "/" is appended to ``code_location``. The code file uploaded to S3 is 'code_location/job-name/source/sourcedir.tar.gz'. If not specified, the default ``code location`` is s3://output_bucket/job-name/. - image_name (str): An alternate image name to use instead of the + image_uri (str): An alternate image name to use instead of the official Sagemaker image for the framework. This is useful to run one of the Sagemaker supported frameworks with an image containing custom dependencies. @@ -1635,6 +1633,8 @@ def __init__( self.git_config = git_config self.source_dir = source_dir self.dependencies = dependencies or [] + self.uploaded_code = None + if enable_cloudwatch_metrics: warnings.warn( "enable_cloudwatch_metrics is now deprecated and will be removed in the future.", @@ -1643,11 +1643,7 @@ def __init__( self.enable_cloudwatch_metrics = False self.container_log_level = container_log_level self.code_location = code_location - self.image_name = image_name - if image_name is not None: - logging.warning(parameter_v2_rename_warning("image_name", "image_uri")) - - self.uploaded_code = None + self.image_uri = image_uri self._hyperparameters = hyperparameters or {} self.checkpoint_s3_uri = checkpoint_s3_uri @@ -1833,8 +1829,8 @@ def train_image(self): Returns: str: The URI of the Docker image. """ - if self.image_name: - return self.image_name + if self.image_uri: + return self.image_uri return create_image_uri( self.sagemaker_session.boto_region_name, self.__framework_name__, diff --git a/src/sagemaker/fw_utils.py b/src/sagemaker/fw_utils.py index d8ceb7240c..1682f13f12 100644 --- a/src/sagemaker/fw_utils.py +++ b/src/sagemaker/fw_utils.py @@ -490,11 +490,11 @@ def _list_files_to_compress(script, directory): return [os.path.join(basedir, name) for name in os.listdir(basedir)] -def framework_name_from_image(image_name): +def framework_name_from_image(image_uri): # noinspection LongLine """Extract the framework and Python version from the image name. Args: - image_name (str): Image URI, which should be one of the following forms: + image_uri (str): Image URI, which should be one of the following forms: legacy: '.dkr.ecr..amazonaws.com/sagemaker---:' legacy: @@ -509,7 +509,7 @@ def framework_name_from_image(image_name): str: If the image is script mode """ sagemaker_pattern = re.compile(ECR_URI_PATTERN) - sagemaker_match = sagemaker_pattern.match(image_name) + sagemaker_match = sagemaker_pattern.match(image_uri) if sagemaker_match is None: return None, None, None, None # extract framework, python version and image tag @@ -691,7 +691,7 @@ def _region_supports_debugger(region_name): return region_name.lower() not in DEBUGGER_UNSUPPORTED_REGIONS -def validate_version_or_image_args(framework_version, py_version, image_name): +def validate_version_or_image_args(framework_version, py_version, image_uri): """Checks if version or image arguments are specified. Validates framework and model arguments to enforce version or image specification. @@ -699,14 +699,14 @@ def validate_version_or_image_args(framework_version, py_version, image_name): Args: framework_version (str): The version of the framework. py_version (str): The version of Python. - image_name (str): The URI of the image. + image_uri (str): The URI of the image. Raises: - ValueError: if `image_name` is None and either `framework_version` or `py_version` is + ValueError: if `image_uri` is None and either `framework_version` or `py_version` is None. """ - if (framework_version is None or py_version is None) and image_name is None: + if (framework_version is None or py_version is None) and image_uri is None: raise ValueError( - "framework_version or py_version was None, yet image_name was also None. " - "Either specify both framework_version and py_version, or specify image_name." + "framework_version or py_version was None, yet image_uri was also None. " + "Either specify both framework_version and py_version, or specify image_uri." ) diff --git a/src/sagemaker/mxnet/estimator.py b/src/sagemaker/mxnet/estimator.py index 3dfb738212..f613327bb3 100644 --- a/src/sagemaker/mxnet/estimator.py +++ b/src/sagemaker/mxnet/estimator.py @@ -44,7 +44,7 @@ def __init__( py_version=None, source_dir=None, hyperparameters=None, - image_name=None, + image_uri=None, distribution=None, **kwargs ): @@ -72,11 +72,11 @@ def __init__( must point to a file located at the root of ``source_dir``. framework_version (str): MXNet version you want to use for executing your model training code. Defaults to `None`. Required unless - ``image_name`` is provided. List of supported versions. + ``image_uri`` is provided. List of supported versions. https://github.com/aws/sagemaker-python-sdk#mxnet-sagemaker-estimators. py_version (str): Python version you want to use for executing your model training code. One of 'py2' or 'py3'. Defaults to ``None``. Required - unless ``image_name`` is provided. + unless ``image_uri`` is provided. source_dir (str): Path (absolute, relative or an S3 URI) to a directory with any other training source code dependencies aside from the entry point file (default: None). If ``source_dir`` is an S3 URI, it must @@ -88,7 +88,7 @@ def __init__( SageMaker. For convenience, this accepts other types for keys and values, but ``str()`` will be called to convert them before training. - image_name (str): If specified, the estimator will use this image for training and + image_uri (str): If specified, the estimator will use this image for training and hosting, instead of selecting the appropriate SageMaker official image based on framework_version and py_version. It can be an ECR url or dockerhub image and tag. @@ -97,7 +97,7 @@ def __init__( * ``custom-image:latest`` If ``framework_version`` or ``py_version`` are ``None``, then - ``image_name`` is required. If also ``None``, then a ``ValueError`` + ``image_uri`` is required. If also ``None``, then a ``ValueError`` will be raised. distribution (dict): A dictionary with information on how to run distributed training (default: None). To have parameter servers launched for training, @@ -111,7 +111,7 @@ def __init__( :class:`~sagemaker.estimator.Framework` and :class:`~sagemaker.estimator.EstimatorBase`. """ - validate_version_or_image_args(framework_version, py_version, image_name) + validate_version_or_image_args(framework_version, py_version, image_uri) if py_version == "py2": logger.warning( python_deprecation_warning(self.__framework_name__, defaults.LATEST_PY2_VERSION) @@ -127,7 +127,7 @@ def __init__( kwargs["enable_sagemaker_metrics"] = True super(MXNet, self).__init__( - entry_point, source_dir, hyperparameters, image_name=image_name, **kwargs + entry_point, source_dir, hyperparameters, image_uri=image_uri, **kwargs ) if distribution is not None: @@ -168,7 +168,7 @@ def create_model( entry_point=None, source_dir=None, dependencies=None, - image_name=None, + image_uri=None, **kwargs ): """Create a SageMaker ``MXNetModel`` object that can be deployed to an @@ -198,7 +198,7 @@ def create_model( any additional libraries that will be exported to the container. If not specified, the dependencies from training are used. This is not supported with "local code" in Local Mode. - image_name (str): If specified, the estimator will use this image for hosting, instead + image_uri (str): If specified, the estimator will use this image for hosting, instead of selecting the appropriate SageMaker official image based on framework_version and py_version. It can be an ECR url or dockerhub image and tag. @@ -214,7 +214,7 @@ def create_model( See :func:`~sagemaker.mxnet.model.MXNetModel` for full details. """ if "image" not in kwargs: - kwargs["image"] = image_name or self.image_name + kwargs["image"] = image_uri or self.image_uri kwargs["name"] = self._get_or_create_name(kwargs.get("name")) @@ -252,8 +252,8 @@ class constructor init_params = super(MXNet, cls)._prepare_init_params_from_job_description( job_details, model_channel_name ) - image_name = init_params.pop("image") - framework, py_version, tag, _ = framework_name_from_image(image_name) + image_uri = init_params.pop("image") + framework, py_version, tag, _ = framework_name_from_image(image_uri) # We switched image tagging scheme from regular image version (e.g. '1.0') to more # expressive containing framework version, device type and python version @@ -271,7 +271,7 @@ class constructor if not framework: # If we were unable to parse the framework name from the image it is not one of our # officially supported images, in this case just add the image to the init params. - init_params["image_name"] = image_name + init_params["image_uri"] = image_uri return init_params if framework != cls.__framework_name__: diff --git a/src/sagemaker/pytorch/estimator.py b/src/sagemaker/pytorch/estimator.py index 3ac8e1511f..6327916cdc 100644 --- a/src/sagemaker/pytorch/estimator.py +++ b/src/sagemaker/pytorch/estimator.py @@ -42,7 +42,7 @@ def __init__( py_version=None, source_dir=None, hyperparameters=None, - image_name=None, + image_uri=None, **kwargs ): """This ``Estimator`` executes an PyTorch script in a managed PyTorch @@ -69,11 +69,11 @@ def __init__( must point to a file located at the root of ``source_dir``. framework_version (str): PyTorch version you want to use for executing your model training code. Defaults to ``None``. Required unless - ``image_name`` is provided. List of supported versions: + ``image_uri`` is provided. List of supported versions: https://github.com/aws/sagemaker-python-sdk#pytorch-sagemaker-estimators. py_version (str): Python version you want to use for executing your model training code. One of 'py2' or 'py3'. Defaults to ``None``. Required - unless ``image_name`` is provided. + unless ``image_uri`` is provided. source_dir (str): Path (absolute, relative or an S3 URI) to a directory with any other training source code dependencies aside from the entry point file (default: None). If ``source_dir`` is an S3 URI, it must @@ -85,7 +85,7 @@ def __init__( SageMaker. For convenience, this accepts other types for keys and values, but ``str()`` will be called to convert them before training. - image_name (str): If specified, the estimator will use this image + image_uri (str): If specified, the estimator will use this image for training and hosting, instead of selecting the appropriate SageMaker official image based on framework_version and py_version. It can be an ECR url or dockerhub image and tag. @@ -95,7 +95,7 @@ def __init__( * ``custom-image:latest`` If ``framework_version`` or ``py_version`` are ``None``, then - ``image_name`` is required. If also ``None``, then a ``ValueError`` + ``image_uri`` is required. If also ``None``, then a ``ValueError`` will be raised. **kwargs: Additional kwargs passed to the :class:`~sagemaker.estimator.Framework` constructor. @@ -106,7 +106,7 @@ def __init__( :class:`~sagemaker.estimator.Framework` and :class:`~sagemaker.estimator.EstimatorBase`. """ - validate_version_or_image_args(framework_version, py_version, image_name) + validate_version_or_image_args(framework_version, py_version, image_uri) if py_version == "py2": logger.warning( python_deprecation_warning(self.__framework_name__, defaults.LATEST_PY2_VERSION) @@ -122,7 +122,7 @@ def __init__( kwargs["enable_sagemaker_metrics"] = True super(PyTorch, self).__init__( - entry_point, source_dir, hyperparameters, image_name=image_name, **kwargs + entry_point, source_dir, hyperparameters, image_uri=image_uri, **kwargs ) def create_model( @@ -168,7 +168,7 @@ def create_model( object. See :func:`~sagemaker.pytorch.model.PyTorchModel` for full details. """ if "image" not in kwargs: - kwargs["image"] = self.image_name + kwargs["image"] = self.image_uri kwargs["name"] = self._get_or_create_name(kwargs.get("name")) @@ -206,8 +206,8 @@ class constructor init_params = super(PyTorch, cls)._prepare_init_params_from_job_description( job_details, model_channel_name ) - image_name = init_params.pop("image") - framework, py_version, tag, _ = framework_name_from_image(image_name) + image_uri = init_params.pop("image") + framework, py_version, tag, _ = framework_name_from_image(image_uri) if tag is None: framework_version = None @@ -219,7 +219,7 @@ class constructor if not framework: # If we were unable to parse the framework name from the image it is not one of our # officially supported images, in this case just add the image to the init params. - init_params["image_name"] = image_name + init_params["image_uri"] = image_uri return init_params if framework != cls.__framework_name__: diff --git a/src/sagemaker/rl/estimator.py b/src/sagemaker/rl/estimator.py index ac8f9eaf08..d2e7cc6f7c 100644 --- a/src/sagemaker/rl/estimator.py +++ b/src/sagemaker/rl/estimator.py @@ -75,7 +75,7 @@ def __init__( framework=None, source_dir=None, hyperparameters=None, - image_name=None, + image_uri=None, metric_definitions=None, **kwargs ): @@ -120,7 +120,7 @@ def __init__( accessible as a dict[str, str] to the training code on SageMaker. For convenience, this accepts other types for keys and values. - image_name (str): An ECR url. If specified, the estimator will use + image_uri (str): An ECR url. If specified, the estimator will use this image for training and hosting, instead of selecting the appropriate SageMaker official image based on framework_version and py_version. Example: @@ -140,9 +140,9 @@ def __init__( :class:`~sagemaker.estimator.Framework` and :class:`~sagemaker.estimator.EstimatorBase`. """ - self._validate_images_args(toolkit, toolkit_version, framework, image_name) + self._validate_images_args(toolkit, toolkit_version, framework, image_uri) - if not image_name: + if not image_uri: self._validate_toolkit_support(toolkit.value, toolkit_version, framework.value) self.toolkit = toolkit.value self.toolkit_version = toolkit_version @@ -159,7 +159,7 @@ def __init__( entry_point, source_dir, hyperparameters, - image_name=image_name, + image_uri=image_uri, metric_definitions=metric_definitions, **kwargs ) @@ -208,20 +208,20 @@ def create_model( sagemaker.model.FrameworkModel: Depending on input parameters returns one of the following: - * :class:`~sagemaker.model.FrameworkModel` - if ``image_name`` is specified + * :class:`~sagemaker.model.FrameworkModel` - if ``image_uri`` is specified on the estimator; - * :class:`~sagemaker.mxnet.MXNetModel` - if ``image_name`` isn't specified and + * :class:`~sagemaker.mxnet.MXNetModel` - if ``image_uri`` isn't specified and MXNet is used as the RL backend; - * :class:`~sagemaker.tensorflow.model.TensorFlowModel` - if ``image_name`` isn't + * :class:`~sagemaker.tensorflow.model.TensorFlowModel` - if ``image_uri`` isn't specified and TensorFlow is used as the RL backend. Raises: - ValueError: If image_name is not specified and framework enum is not valid. + ValueError: If image_uri is not specified and framework enum is not valid. """ base_args = dict( model_data=self.model_data, role=role or self.role, - image=kwargs.get("image", self.image_name), + image=kwargs.get("image", self.image_uri), container_log_level=self.container_log_level, sagemaker_session=self.sagemaker_session, vpc_config=self.get_vpc_config(vpc_config_override), @@ -245,7 +245,7 @@ def create_model( ) extended_args.update(base_args) - if self.image_name: + if self.image_uri: return FrameworkModel(**extended_args) if self.toolkit == RLToolkit.RAY.value: @@ -275,8 +275,8 @@ def train_image(self): Returns: str: The URI of the Docker image. """ - if self.image_name: - return self.image_name + if self.image_uri: + return self.image_uri return fw_utils.create_image_uri( self.sagemaker_session.boto_region_name, self._image_framework(), @@ -303,13 +303,13 @@ class constructor job_details, model_channel_name ) - image_name = init_params.pop("image") - framework, _, tag, _ = fw_utils.framework_name_from_image(image_name) + image_uri = init_params.pop("image") + framework, _, tag, _ = fw_utils.framework_name_from_image(image_uri) if not framework: # If we were unable to parse the framework name from the image it is not one of our # officially supported images, in this case just add the image to the init params. - init_params["image_name"] = image_name + init_params["image_uri"] = image_uri return init_params toolkit, toolkit_version = cls._toolkit_and_version_from_tag(tag) @@ -381,18 +381,18 @@ def _validate_toolkit_format(cls, toolkit): ) @classmethod - def _validate_images_args(cls, toolkit, toolkit_version, framework, image_name): + def _validate_images_args(cls, toolkit, toolkit_version, framework, image_uri): """ Args: toolkit: toolkit_version: framework: - image_name: + image_uri: """ cls._validate_toolkit_format(toolkit) cls._validate_framework_format(framework) - if not image_name: + if not image_uri: not_found_args = [] if not toolkit: not_found_args.append("toolkit") @@ -402,7 +402,7 @@ def _validate_images_args(cls, toolkit, toolkit_version, framework, image_name): not_found_args.append("framework") if not_found_args: raise AttributeError( - "Please provide `{}` or `image_name` parameter.".format( + "Please provide `{}` or `image_uri` parameter.".format( "`, `".join(not_found_args) ) ) @@ -416,7 +416,7 @@ def _validate_images_args(cls, toolkit, toolkit_version, framework, image_name): found_args.append("framework") if found_args: logger.warning( - "Parameter `image_name` is specified, " + "Parameter `image_uri` is specified, " "`%s` are going to be ignored when choosing the image.", "`, `".join(found_args), ) diff --git a/src/sagemaker/sklearn/estimator.py b/src/sagemaker/sklearn/estimator.py index 7d1d255f69..0cf9be61dd 100644 --- a/src/sagemaker/sklearn/estimator.py +++ b/src/sagemaker/sklearn/estimator.py @@ -41,7 +41,7 @@ def __init__( py_version="py3", source_dir=None, hyperparameters=None, - image_name=None, + image_uri=None, **kwargs ): """This ``Estimator`` executes an Scikit-learn script in a managed @@ -69,11 +69,11 @@ def __init__( must point to a file located at the root of ``source_dir``. framework_version (str): Scikit-learn version you want to use for executing your model training code. Defaults to ``None``. Required - unless ``image_name`` is provided. List of supported versions: + unless ``image_uri`` is provided. List of supported versions: https://github.com/aws/sagemaker-python-sdk#sklearn-sagemaker-estimators py_version (str): Python version you want to use for executing your model training code (default: 'py3'). Currently, 'py3' is the only - supported version. If ``None`` is passed in, ``image_name`` must be + supported version. If ``None`` is passed in, ``image_uri`` must be provided. source_dir (str): Path (absolute, relative or an S3 URI) to a directory with any other training source code dependencies aside from the entry @@ -86,7 +86,7 @@ def __init__( SageMaker. For convenience, this accepts other types for keys and values, but ``str()`` will be called to convert them before training. - image_name (str): If specified, the estimator will use this image + image_uri (str): If specified, the estimator will use this image for training and hosting, instead of selecting the appropriate SageMaker official image based on framework_version and py_version. It can be an ECR url or dockerhub image and tag. @@ -96,7 +96,7 @@ def __init__( custom-image:latest. If ``framework_version`` or ``py_version`` are ``None``, then - ``image_name`` is required. If also ``None``, then a ``ValueError`` + ``image_uri`` is required. If also ``None``, then a ``ValueError`` will be raised. **kwargs: Additional kwargs passed to the :class:`~sagemaker.estimator.Framework` constructor. @@ -107,7 +107,7 @@ def __init__( :class:`~sagemaker.estimator.Framework` and :class:`~sagemaker.estimator.EstimatorBase`. """ - validate_version_or_image_args(framework_version, py_version, image_name) + validate_version_or_image_args(framework_version, py_version, image_uri) if py_version and py_version != "py3": raise AttributeError( "Scikit-learn image only supports Python 3. Please use 'py3' for py_version." @@ -133,13 +133,13 @@ def __init__( entry_point, source_dir, hyperparameters, - image_name=image_name, + image_uri=image_uri, **dict(kwargs, train_instance_count=1) ) - if image_name is None: + if image_uri is None: image_tag = "{}-{}-{}".format(framework_version, "cpu", py_version) - self.image_name = default_framework_uri( + self.image_uri = default_framework_uri( SKLearn.__framework_name__, self.sagemaker_session.boto_region_name, image_tag ) @@ -189,7 +189,7 @@ def create_model( kwargs["name"] = self._get_or_create_name(kwargs.get("name")) if "image" not in kwargs: - kwargs["image"] = self.image_name + kwargs["image"] = self.image_uri if "enable_network_isolation" not in kwargs: kwargs["enable_network_isolation"] = self.enable_network_isolation() @@ -228,8 +228,8 @@ class constructor init_params = super(SKLearn, cls)._prepare_init_params_from_job_description( job_details, model_channel_name ) - image_name = init_params.pop("image") - framework, py_version, tag, _ = framework_name_from_image(image_name) + image_uri = init_params.pop("image") + framework, py_version, tag, _ = framework_name_from_image(image_uri) if tag is None: framework_version = None @@ -241,7 +241,7 @@ class constructor if not framework: # If we were unable to parse the framework name from the image it is not one of our # officially supported images, in this case just add the image to the init params. - init_params["image_name"] = image_name + init_params["image_uri"] = image_uri return init_params if framework and framework != cls.__framework_name__: diff --git a/src/sagemaker/tensorflow/estimator.py b/src/sagemaker/tensorflow/estimator.py index 5be0a4f37a..49eba2100c 100644 --- a/src/sagemaker/tensorflow/estimator.py +++ b/src/sagemaker/tensorflow/estimator.py @@ -44,7 +44,7 @@ def __init__( py_version=None, framework_version=None, model_dir=None, - image_name=None, + image_uri=None, distribution=None, **kwargs ): @@ -52,9 +52,9 @@ def __init__( Args: py_version (str): Python version you want to use for executing your model training - code. Defaults to ``None``. Required unless ``image_name`` is provided. + code. Defaults to ``None``. Required unless ``image_uri`` is provided. framework_version (str): TensorFlow version you want to use for executing your model - training code. Defaults to ``None``. Required unless ``image_name`` is provided. + training code. Defaults to ``None``. Required unless ``image_uri`` is provided. List of supported versions: https://github.com/aws/sagemaker-python-sdk#tensorflow-sagemaker-estimators. model_dir (str): S3 location where the checkpoint data and models can be exported to @@ -70,7 +70,7 @@ def __init__( To disable having ``model_dir`` passed to your training script, set ``model_dir=False``. - image_name (str): If specified, the estimator will use this image for training and + image_uri (str): If specified, the estimator will use this image for training and hosting, instead of selecting the appropriate SageMaker official image based on framework_version and py_version. It can be an ECR url or dockerhub image and tag. @@ -79,7 +79,7 @@ def __init__( custom-image:latest. If ``framework_version`` or ``py_version`` are ``None``, then - ``image_name`` is required. If also ``None``, then a ``ValueError`` + ``image_uri`` is required. If also ``None``, then a ``ValueError`` will be raised. distribution (dict): A dictionary with information on how to run distributed training (default: None). Currently we support distributed training with parameter servers @@ -114,7 +114,7 @@ def __init__( :class:`~sagemaker.estimator.Framework` and :class:`~sagemaker.estimator.EstimatorBase`. """ - fw.validate_version_or_image_args(framework_version, py_version, image_name) + fw.validate_version_or_image_args(framework_version, py_version, image_uri) if py_version == "py2": logger.warning( fw.python_deprecation_warning(self.__framework_name__, defaults.LATEST_PY2_VERSION) @@ -133,7 +133,7 @@ def __init__( if framework_version and fw.is_version_equal_or_higher([1, 15], framework_version): kwargs["enable_sagemaker_metrics"] = True - super(TensorFlow, self).__init__(image_name=image_name, **kwargs) + super(TensorFlow, self).__init__(image_uri=image_uri, **kwargs) self.model_dir = model_dir self.distribution = distribution or {} @@ -150,7 +150,7 @@ def _validate_args(self, py_version): ) raise AttributeError(msg) - if self.image_name is None and self._only_legacy_mode_supported(): + if self.image_uri is None and self._only_legacy_mode_supported(): legacy_image_uri = fw.create_image_uri( self.sagemaker_session.boto_region_name, "tensorflow", @@ -161,7 +161,7 @@ def _validate_args(self, py_version): msg = ( "TF {} supports only legacy mode. Please supply the image URI directly with " - "'image_name={}' and set 'model_dir=False'. If you are using any legacy parameters " + "'image_uri={}' and set 'model_dir=False'. If you are using any legacy parameters " "(training_steps, evaluation_steps, checkpoint_path, requirements_file), " "make sure to pass them directly as hyperparameters instead. For more, see " "https://sagemaker.readthedocs.io/en/v2.0.0.rc0/frameworks/tensorflow/upgrade_from_legacy.html." @@ -192,12 +192,12 @@ def _prepare_init_params_from_job_description(cls, job_details, model_channel_na job_details, model_channel_name ) - image_name = init_params.pop("image") - framework, py_version, tag, script_mode = fw.framework_name_from_image(image_name) + image_uri = init_params.pop("image") + framework, py_version, tag, script_mode = fw.framework_name_from_image(image_uri) if not framework: # If we were unable to parse the framework name from the image, it is not one of our # officially supported images, so just add the image to the init params. - init_params["image_name"] = image_name + init_params["image_uri"] = image_uri return init_params model_dir = init_params["hyperparameters"].pop("model_dir", None) @@ -218,7 +218,7 @@ def _prepare_init_params_from_job_description(cls, job_details, model_channel_na # Legacy images are required to be passed in explicitly. if not script_mode: - init_params["image_name"] = image_name + init_params["image_uri"] = image_uri if framework != cls.__framework_name__: raise ValueError( @@ -271,7 +271,7 @@ def create_model( kwargs["name"] = self._get_or_create_name(kwargs.get("name")) if "image" not in kwargs: - kwargs["image"] = self.image_name + kwargs["image"] = self.image_uri if "enable_network_isolation" not in kwargs: kwargs["enable_network_isolation"] = self.enable_network_isolation() @@ -356,8 +356,8 @@ def _validate_and_set_debugger_configs(self): def train_image(self): """Placeholder docstring""" - if self.image_name: - return self.image_name + if self.image_uri: + return self.image_uri return fw.create_image_uri( self.sagemaker_session.boto_region_name, diff --git a/src/sagemaker/tensorflow/model.py b/src/sagemaker/tensorflow/model.py index 17aedfffa1..2214f73c9b 100644 --- a/src/sagemaker/tensorflow/model.py +++ b/src/sagemaker/tensorflow/model.py @@ -181,7 +181,7 @@ def __init__( if framework_version is None and image is None: raise ValueError( "Both framework_version and image were None. " - "Either specify framework_version or specify image_name." + "Either specify framework_version or specify image." ) self.framework_version = framework_version diff --git a/src/sagemaker/transformer.py b/src/sagemaker/transformer.py index 95cbb2e634..83c6a099e7 100644 --- a/src/sagemaker/transformer.py +++ b/src/sagemaker/transformer.py @@ -219,14 +219,14 @@ def delete_model(self): def _retrieve_base_name(self): """Placeholder docstring""" - image_name = self._retrieve_image_name() + image_uri = self._retrieve_image_uri() - if image_name: - return base_name_from_image(image_name) + if image_uri: + return base_name_from_image(image_uri) return self.model_name - def _retrieve_image_name(self): + def _retrieve_image_uri(self): """Placeholder docstring""" try: model_desc = self.sagemaker_session.sagemaker_client.describe_model( diff --git a/src/sagemaker/tuner.py b/src/sagemaker/tuner.py index 9ab8b75c46..f91b853ed5 100644 --- a/src/sagemaker/tuner.py +++ b/src/sagemaker/tuner.py @@ -858,8 +858,8 @@ def _prepare_estimator_cls(cls, estimator_cls, training_details): return getattr(importlib.import_module(json.loads(module)), json.loads(cls_name)) # Then try to derive the estimator from the image name for 1P algorithms - image_name = training_details["AlgorithmSpecification"]["TrainingImage"] - algorithm = image_name[image_name.find("/") + 1 : image_name.find(":")] + image_uri = training_details["AlgorithmSpecification"]["TrainingImage"] + algorithm = image_uri[image_uri.find("/") + 1 : image_uri.find(":")] if algorithm in AMAZON_ESTIMATOR_CLS_NAMES: cls_name = AMAZON_ESTIMATOR_CLS_NAMES[algorithm] return getattr(importlib.import_module(AMAZON_ESTIMATOR_MODULE), cls_name) diff --git a/src/sagemaker/xgboost/README.rst b/src/sagemaker/xgboost/README.rst index 9ccf169fc5..2dd3ac98c9 100644 --- a/src/sagemaker/xgboost/README.rst +++ b/src/sagemaker/xgboost/README.rst @@ -79,6 +79,6 @@ minor version, which will cause your training script to be run on the latest sup version. Alternatively, you can build your own image by following the instructions in the SageMaker XGBoost containers -repository, and passing ``image_name`` to the XGBoost Estimator constructor. +repository, and passing ``image_uri`` to the XGBoost Estimator constructor. You can visit the SageMaker XGBoost containers repository here: https://github.com/aws/sagemaker-xgboost-container diff --git a/src/sagemaker/xgboost/estimator.py b/src/sagemaker/xgboost/estimator.py index d0b02fa857..af0f5f2d08 100644 --- a/src/sagemaker/xgboost/estimator.py +++ b/src/sagemaker/xgboost/estimator.py @@ -50,7 +50,7 @@ def __init__( source_dir=None, hyperparameters=None, py_version="py3", - image_name=None, + image_uri=None, **kwargs ): """ @@ -85,7 +85,7 @@ def __init__( ``str()`` will be called to convert them before training. py_version (str): Python version you want to use for executing your model training code (default: 'py3'). - image_name (str): If specified, the estimator will use this image for training and + image_uri (str): If specified, the estimator will use this image for training and hosting, instead of selecting the appropriate SageMaker official image based on framework_version and py_version. It can be an ECR url or dockerhub image and tag. @@ -102,7 +102,7 @@ def __init__( :class:`~sagemaker.estimator.EstimatorBase`. """ super(XGBoost, self).__init__( - entry_point, source_dir, hyperparameters, image_name=image_name, **kwargs + entry_point, source_dir, hyperparameters, image_uri=image_uri, **kwargs ) if py_version == "py2": @@ -118,8 +118,8 @@ def __init__( ) ) - if image_name is None: - self.image_name = get_xgboost_image_uri( + if image_uri is None: + self.image_uri = get_xgboost_image_uri( self.sagemaker_session.boto_region_name, framework_version ) @@ -167,7 +167,7 @@ def create_model( kwargs["name"] = self._get_or_create_name(kwargs.get("name")) if "image" not in kwargs: - kwargs["image"] = self.image_name + kwargs["image"] = self.image_uri return XGBoostModel( self.model_data, @@ -261,8 +261,8 @@ def _prepare_init_params_from_job_description(cls, job_details, model_channel_na """ init_params = super(XGBoost, cls)._prepare_init_params_from_job_description(job_details) - image_name = init_params.pop("image") - framework, py_version, tag, _ = framework_name_from_image(image_name) + image_uri = init_params.pop("image") + framework, py_version, tag, _ = framework_name_from_image(image_uri) init_params["py_version"] = py_version if framework and framework != cls.__framework_name__: @@ -276,5 +276,5 @@ def _prepare_init_params_from_job_description(cls, job_details, model_channel_na if not framework: # If we were unable to parse the framework name from the image it is not one of our # officially supported images, in this case just add the image to the init params. - init_params["image_name"] = image_name + init_params["image_uri"] = image_uri return init_params diff --git a/tests/integ/test_airflow_config.py b/tests/integ/test_airflow_config.py index e1af3fd5a6..0c55de2f06 100644 --- a/tests/integ/test_airflow_config.py +++ b/tests/integ/test_airflow_config.py @@ -73,7 +73,7 @@ def test_byo_airflow_config_uploads_data_source_to_s3_when_inputs_provided( ) estimator = Estimator( - image_name=get_image_uri( + image_uri=get_image_uri( sagemaker_session.boto_session.region_name, "factorization-machines" ), role=ROLE, @@ -516,7 +516,7 @@ def test_tf_airflow_config_uploads_data_source_to_s3( ): with timeout(seconds=AIRFLOW_CONFIG_TIMEOUT_IN_SECONDS): tf = TensorFlow( - image_name=get_image_uri( + image_uri=get_image_uri( sagemaker_session.boto_session.region_name, "factorization-machines" ), entry_point=SCRIPT, diff --git a/tests/integ/test_byo_estimator.py b/tests/integ/test_byo_estimator.py index 738af4c57a..00a69a5722 100644 --- a/tests/integ/test_byo_estimator.py +++ b/tests/integ/test_byo_estimator.py @@ -54,7 +54,7 @@ def test_byo_estimator(sagemaker_session, region, cpu_instance_type, training_se Default predictor is updated with json serializer and deserializer. """ - image_name = get_image_uri(region, "factorization-machines") + image_uri = get_image_uri(region, "factorization-machines") training_data_path = os.path.join(DATA_DIR, "dummy_tensor") job_name = unique_name_from_base("byo") @@ -67,7 +67,7 @@ def test_byo_estimator(sagemaker_session, region, cpu_instance_type, training_se ) estimator = Estimator( - image_name=image_name, + image_uri=image_uri, role="SageMakerRole", train_instance_count=1, train_instance_type=cpu_instance_type, @@ -96,7 +96,7 @@ def test_byo_estimator(sagemaker_session, region, cpu_instance_type, training_se def test_async_byo_estimator(sagemaker_session, region, cpu_instance_type, training_set): - image_name = get_image_uri(region, "factorization-machines") + image_uri = get_image_uri(region, "factorization-machines") endpoint_name = unique_name_from_base("byo") training_data_path = os.path.join(DATA_DIR, "dummy_tensor") job_name = unique_name_from_base("byo") @@ -110,7 +110,7 @@ def test_async_byo_estimator(sagemaker_session, region, cpu_instance_type, train ) estimator = Estimator( - image_name=image_name, + image_uri=image_uri, role="SageMakerRole", train_instance_count=1, train_instance_type=cpu_instance_type, @@ -140,4 +140,4 @@ def test_async_byo_estimator(sagemaker_session, region, cpu_instance_type, train for prediction in result["predictions"]: assert prediction["score"] is not None - assert estimator.train_image() == image_name + assert estimator.train_image() == image_uri diff --git a/tests/integ/test_multidatamodel.py b/tests/integ/test_multidatamodel.py index 0588c8c74d..0864c45b20 100644 --- a/tests/integ/test_multidatamodel.py +++ b/tests/integ/test_multidatamodel.py @@ -346,7 +346,7 @@ def _mxnet_training_job( # Replace the container image value for now since the frameworks do not support # multi-model container image yet. - return mx.create_model(image_name=container_image) + return mx.create_model(image_uri=container_image) def test_multi_data_model_deploy_train_model_from_amazon_first_party_estimator( diff --git a/tests/integ/test_tuner.py b/tests/integ/test_tuner.py index 58e2eed383..03c31bdd0f 100644 --- a/tests/integ/test_tuner.py +++ b/tests/integ/test_tuner.py @@ -843,7 +843,7 @@ def test_tuning_byo_estimator(sagemaker_session, cpu_instance_type): Later the trained model is deployed and prediction is called against the endpoint. Default predictor is updated with json serializer and deserializer. """ - image_name = get_image_uri(sagemaker_session.boto_session.region_name, "factorization-machines") + image_uri = get_image_uri(sagemaker_session.boto_session.region_name, "factorization-machines") training_data_path = os.path.join(DATA_DIR, "dummy_tensor") with timeout(minutes=TUNING_DEFAULT_TIMEOUT_MINUTES): @@ -854,7 +854,7 @@ def test_tuning_byo_estimator(sagemaker_session, cpu_instance_type): ) estimator = Estimator( - image_name=image_name, + image_uri=image_uri, role="SageMakerRole", train_instance_count=1, train_instance_type=cpu_instance_type, diff --git a/tests/integ/test_tuner_multi_algo.py b/tests/integ/test_tuner_multi_algo.py index ad933075e1..40312e8c51 100644 --- a/tests/integ/test_tuner_multi_algo.py +++ b/tests/integ/test_tuner_multi_algo.py @@ -68,7 +68,7 @@ def estimator_fm(sagemaker_session, cpu_instance_type): ) estimator = Estimator( - image_name=fm_image, + image_uri=fm_image, role=EXECUTION_ROLE, train_instance_count=1, train_instance_type=cpu_instance_type, @@ -87,7 +87,7 @@ def estimator_knn(sagemaker_session, cpu_instance_type): knn_image = get_image_uri(sagemaker_session.boto_session.region_name, "knn", repo_version="1") estimator = Estimator( - image_name=knn_image, + image_uri=knn_image, role=EXECUTION_ROLE, train_instance_count=1, train_instance_type=cpu_instance_type, diff --git a/tests/unit/sagemaker/tensorflow/test_estimator.py b/tests/unit/sagemaker/tensorflow/test_estimator.py index c3f39c29b6..fd2ad7bf87 100644 --- a/tests/unit/sagemaker/tensorflow/test_estimator.py +++ b/tests/unit/sagemaker/tensorflow/test_estimator.py @@ -262,7 +262,7 @@ def test_create_model_with_custom_image(sagemaker_session): sagemaker_session=sagemaker_session, train_instance_count=INSTANCE_COUNT, train_instance_type=INSTANCE_TYPE, - image_name=custom_image, + image_uri=custom_image, container_log_level=container_log_level, base_job_name="job", source_dir=source_dir, @@ -518,5 +518,5 @@ def test_hyperparameters_no_model_dir(sagemaker_session, tf_version, tf_py_versi def test_train_image_custom_image(sagemaker_session): custom_image = "tensorflow:latest" - tf = _build_tf(sagemaker_session, image_name=custom_image) + tf = _build_tf(sagemaker_session, image_uri=custom_image) assert custom_image == tf.train_image() diff --git a/tests/unit/sagemaker/tensorflow/test_estimator_attach.py b/tests/unit/sagemaker/tensorflow/test_estimator_attach.py index 9b571fe7ce..5adafba10e 100644 --- a/tests/unit/sagemaker/tensorflow/test_estimator_attach.py +++ b/tests/unit/sagemaker/tensorflow/test_estimator_attach.py @@ -246,7 +246,7 @@ def test_attach_custom_image(sagemaker_session): ) estimator = TensorFlow.attach(training_job_name="neo", sagemaker_session=sagemaker_session) - assert estimator.image_name == training_image + assert estimator.image_uri == training_image assert estimator.train_image() == training_image diff --git a/tests/unit/sagemaker/tensorflow/test_estimator_init.py b/tests/unit/sagemaker/tensorflow/test_estimator_init.py index 86b6df3065..c9297e7556 100644 --- a/tests/unit/sagemaker/tensorflow/test_estimator_init.py +++ b/tests/unit/sagemaker/tensorflow/test_estimator_init.py @@ -98,7 +98,7 @@ def test_disable_sm_metrics_if_fw_ver_is_less_than_1_15( sagemaker_session, framework_version=tf_version, py_version=tf_py_version, - image_name="old-image", + image_uri="old-image", ) assert tf.enable_sagemaker_metrics is None @@ -111,7 +111,7 @@ def test_enable_sm_metrics_if_fw_ver_is_at_least_1_15(sagemaker_session, tf_vers assert tf.enable_sagemaker_metrics -def test_require_image_name_if_fw_ver_is_less_than_1_11( +def test_require_image_uri_if_fw_ver_is_less_than_1_11( sagemaker_session, tf_version, tf_py_version ): if version.Version(tf_version) > version.Version("1.10"): @@ -122,7 +122,7 @@ def test_require_image_name_if_fw_ver_is_less_than_1_11( expected_msg = ( "TF {version} supports only legacy mode. Please supply the image URI directly with " - "'image_name=520713654638.dkr.ecr.{region}.amazonaws.com/" + "'image_uri=520713654638.dkr.ecr.{region}.amazonaws.com/" "sagemaker-tensorflow:{version}-cpu-py2' and set 'model_dir=False'. " "If you are using any legacy parameters (training_steps, evaluation_steps, " "checkpoint_path, requirements_file), make sure to pass them directly as hyperparameters instead." diff --git a/tests/unit/sagemaker/tensorflow/test_tfs.py b/tests/unit/sagemaker/tensorflow/test_tfs.py index c9ac4516c9..57d0da56aa 100644 --- a/tests/unit/sagemaker/tensorflow/test_tfs.py +++ b/tests/unit/sagemaker/tensorflow/test_tfs.py @@ -274,7 +274,7 @@ def test_estimator_deploy(sagemaker_session): sagemaker_session=sagemaker_session, train_instance_count=INSTANCE_COUNT, train_instance_type=INSTANCE_TYPE, - image_name=custom_image, + image_uri=custom_image, container_log_level=container_log_level, base_job_name="job", source_dir=source_dir, diff --git a/tests/unit/test_airflow.py b/tests/unit/test_airflow.py index c29fb317d2..658368a3a0 100644 --- a/tests/unit/test_airflow.py +++ b/tests/unit/test_airflow.py @@ -46,7 +46,7 @@ def sagemaker_session(): @patch("sagemaker.utils.sagemaker_timestamp", MagicMock(return_value=TIME_STAMP)) def test_byo_training_config_required_args(sagemaker_session): byo = estimator.Estimator( - image_name="byo", + image_uri="byo", role="{{ role }}", train_instance_count="{{ instance_count }}", train_instance_type="ml.c4.2xlarge", @@ -89,7 +89,7 @@ def test_byo_training_config_required_args(sagemaker_session): @patch("sagemaker.utils.sagemaker_timestamp", MagicMock(return_value=TIME_STAMP)) def test_byo_training_config_all_args(sagemaker_session): byo = estimator.Estimator( - image_name="byo", + image_uri="byo", role="{{ role }}", train_instance_count="{{ instance_count }}", train_instance_type="ml.c4.2xlarge", diff --git a/tests/unit/test_chainer.py b/tests/unit/test_chainer.py index 2906cdd4d9..ed028404e2 100644 --- a/tests/unit/test_chainer.py +++ b/tests/unit/test_chainer.py @@ -36,8 +36,8 @@ INSTANCE_COUNT = 1 INSTANCE_TYPE = "ml.c4.4xlarge" ACCELERATOR_TYPE = "ml.eia.medium" -IMAGE_NAME = "sagemaker-chainer" -JOB_NAME = "{}-{}".format(IMAGE_NAME, TIMESTAMP) +IMAGE_URI = "sagemaker-chainer" +JOB_NAME = "{}-{}".format(IMAGE_URI, TIMESTAMP) IMAGE_URI_FORMAT_STRING = "520713654638.dkr.ecr.{}.amazonaws.com/{}:{}-{}-{}" ROLE = "Dummy" REGION = "us-west-2" @@ -75,11 +75,11 @@ def sagemaker_session(): def _get_full_cpu_image_uri(version, py_version): - return IMAGE_URI_FORMAT_STRING.format(REGION, IMAGE_NAME, version, "cpu", py_version) + return IMAGE_URI_FORMAT_STRING.format(REGION, IMAGE_URI, version, "cpu", py_version) def _get_full_gpu_image_uri(version, py_version): - return IMAGE_URI_FORMAT_STRING.format(REGION, IMAGE_NAME, version, "gpu", py_version) + return IMAGE_URI_FORMAT_STRING.format(REGION, IMAGE_URI, version, "gpu", py_version) def _get_full_cpu_image_uri_with_ei(version, py_version): @@ -320,7 +320,7 @@ def test_create_model_with_custom_image(sagemaker_session): sagemaker_session=sagemaker_session, train_instance_count=INSTANCE_COUNT, train_instance_type=INSTANCE_TYPE, - image_name=custom_image, + image_uri=custom_image, container_log_level=container_log_level, base_job_name="job", source_dir=source_dir, @@ -604,7 +604,7 @@ def test_attach_custom_image(sagemaker_session): ) estimator = Chainer.attach(training_job_name="neo", sagemaker_session=sagemaker_session) - assert estimator.image_name == training_image + assert estimator.image_uri == training_image assert estimator.train_image() == training_image diff --git a/tests/unit/test_estimator.py b/tests/unit/test_estimator.py index bab5d39e1a..6e814fc6fc 100644 --- a/tests/unit/test_estimator.py +++ b/tests/unit/test_estimator.py @@ -45,9 +45,9 @@ INSTANCE_TYPE = "c4.4xlarge" ACCELERATOR_TYPE = "ml.eia.medium" ROLE = "DummyRole" -IMAGE_NAME = "fakeimage" +IMAGE_URI = "fakeimage" REGION = "us-west-2" -JOB_NAME = "{}-{}".format(IMAGE_NAME, TIMESTAMP) +JOB_NAME = "{}-{}".format(IMAGE_URI, TIMESTAMP) TAGS = [{"Name": "some-tag", "Value": "value-for-tag"}] OUTPUT_PATH = "s3://bucket/prefix" GIT_REPO = "https://github.com/aws/sagemaker-python-sdk.git" @@ -113,7 +113,7 @@ class DummyFramework(Framework): __framework_name__ = "dummy" def train_image(self): - return IMAGE_NAME + return IMAGE_URI def create_model( self, @@ -577,7 +577,7 @@ def test_local_code_location(): sagemaker_session=sms, train_instance_count=1, train_instance_type="local", - base_job_name=IMAGE_NAME, + base_job_name=IMAGE_URI, hyperparameters={123: [456], "learning_rate": 0.1}, ) @@ -596,7 +596,7 @@ def test_start_new_convert_hyperparameters_to_str(strftime, sagemaker_session): sagemaker_session=sagemaker_session, train_instance_count=INSTANCE_COUNT, train_instance_type=INSTANCE_TYPE, - base_job_name=IMAGE_NAME, + base_job_name=IMAGE_URI, hyperparameters={123: [456], "learning_rate": 0.1}, ) t.fit("s3://{}".format(uri)) @@ -790,7 +790,7 @@ def test_fit_verify_job_name(strftime, sagemaker_session): _, _, train_kwargs = sagemaker_session.train.mock_calls[0] assert train_kwargs["hyperparameters"]["sagemaker_enable_cloudwatch_metrics"] - assert train_kwargs["image"] == IMAGE_NAME + assert train_kwargs["image"] == IMAGE_URI assert train_kwargs["input_mode"] == "File" assert train_kwargs["tags"] == TAGS assert train_kwargs["job_name"] == JOB_NAME @@ -1306,7 +1306,7 @@ def test_framework_transformer_creation(name_from_base, sagemaker_session): transformer = fw.transformer(INSTANCE_COUNT, INSTANCE_TYPE) - name_from_base.assert_called_with(IMAGE_NAME) + name_from_base.assert_called_with(IMAGE_URI) sagemaker_session.create_model.assert_called_with( MODEL_IMAGE, ROLE, @@ -1426,7 +1426,7 @@ def test_ensure_latest_training_job_failure(sagemaker_session): @patch("sagemaker.estimator.name_from_base") def test_estimator_transformer_creation(name_from_base, create_model, sagemaker_session): estimator = Estimator( - image_name=IMAGE_NAME, + image_uri=IMAGE_URI, role=ROLE, train_instance_count=INSTANCE_COUNT, train_instance_type=INSTANCE_TYPE, @@ -1458,7 +1458,7 @@ def test_estimator_transformer_creation_with_optional_params(create_model, sagem kms_key = "key" estimator = Estimator( - image_name=IMAGE_NAME, + image_uri=IMAGE_URI, role=ROLE, train_instance_count=INSTANCE_COUNT, train_instance_type=INSTANCE_TYPE, @@ -1519,7 +1519,7 @@ def test_start_new(sagemaker_session): inputs = "s3://mybucket/train" estimator = Estimator( - IMAGE_NAME, + IMAGE_URI, ROLE, INSTANCE_COUNT, INSTANCE_TYPE, @@ -1544,7 +1544,7 @@ def test_start_new_not_local_mode_error(sagemaker_session): inputs = "file://mybucket/train" estimator = Estimator( - IMAGE_NAME, + IMAGE_URI, ROLE, INSTANCE_COUNT, INSTANCE_TYPE, @@ -1665,7 +1665,7 @@ def test_unsupported_type_in_dict(): NO_INPUT_TRAIN_CALL = { "hyperparameters": {}, - "image": IMAGE_NAME, + "image": IMAGE_URI, "input_config": None, "input_mode": "File", "output_config": {"S3OutputPath": OUTPUT_PATH}, @@ -1718,7 +1718,7 @@ def test_unsupported_type_in_dict(): def test_fit_deploy_tags_in_estimator(name_from_base, sagemaker_session): tags = [{"Key": "TagtestKey", "Value": "TagtestValue"}] estimator = Estimator( - IMAGE_NAME, + IMAGE_URI, ROLE, INSTANCE_COUNT, INSTANCE_TYPE, @@ -1743,7 +1743,7 @@ def test_fit_deploy_tags_in_estimator(name_from_base, sagemaker_session): } ] - name_from_base.assert_called_with(IMAGE_NAME) + name_from_base.assert_called_with(IMAGE_URI) sagemaker_session.endpoint_from_production_variants.assert_called_with( name=model_name, @@ -1767,7 +1767,7 @@ def test_fit_deploy_tags_in_estimator(name_from_base, sagemaker_session): @patch("sagemaker.estimator.name_from_base") def test_fit_deploy_tags(name_from_base, sagemaker_session): estimator = Estimator( - IMAGE_NAME, ROLE, INSTANCE_COUNT, INSTANCE_TYPE, sagemaker_session=sagemaker_session + IMAGE_URI, ROLE, INSTANCE_COUNT, INSTANCE_TYPE, sagemaker_session=sagemaker_session ) estimator.fit() @@ -1788,7 +1788,7 @@ def test_fit_deploy_tags(name_from_base, sagemaker_session): } ] - name_from_base.assert_called_with(IMAGE_NAME) + name_from_base.assert_called_with(IMAGE_URI) sagemaker_session.endpoint_from_production_variants.assert_called_with( name=model_name, @@ -1811,7 +1811,7 @@ def test_fit_deploy_tags(name_from_base, sagemaker_session): def test_generic_to_fit_no_input(sagemaker_session): e = Estimator( - IMAGE_NAME, + IMAGE_URI, ROLE, INSTANCE_COUNT, INSTANCE_TYPE, @@ -1824,7 +1824,7 @@ def test_generic_to_fit_no_input(sagemaker_session): sagemaker_session.train.assert_called_once() assert len(sagemaker_session.train.call_args[0]) == 0 args = sagemaker_session.train.call_args[1] - assert args["job_name"].startswith(IMAGE_NAME) + assert args["job_name"].startswith(IMAGE_URI) args.pop("job_name") args.pop("role") @@ -1834,7 +1834,7 @@ def test_generic_to_fit_no_input(sagemaker_session): def test_generic_to_fit_no_hps(sagemaker_session): e = Estimator( - IMAGE_NAME, + IMAGE_URI, ROLE, INSTANCE_COUNT, INSTANCE_TYPE, @@ -1847,7 +1847,7 @@ def test_generic_to_fit_no_hps(sagemaker_session): sagemaker_session.train.assert_called_once() assert len(sagemaker_session.train.call_args[0]) == 0 args = sagemaker_session.train.call_args[1] - assert args["job_name"].startswith(IMAGE_NAME) + assert args["job_name"].startswith(IMAGE_URI) args.pop("job_name") args.pop("role") @@ -1857,7 +1857,7 @@ def test_generic_to_fit_no_hps(sagemaker_session): def test_generic_to_fit_with_hps(sagemaker_session): e = Estimator( - IMAGE_NAME, + IMAGE_URI, ROLE, INSTANCE_COUNT, INSTANCE_TYPE, @@ -1872,7 +1872,7 @@ def test_generic_to_fit_with_hps(sagemaker_session): sagemaker_session.train.assert_called_once() assert len(sagemaker_session.train.call_args[0]) == 0 args = sagemaker_session.train.call_args[1] - assert args["job_name"].startswith(IMAGE_NAME) + assert args["job_name"].startswith(IMAGE_URI) args.pop("job_name") args.pop("role") @@ -1882,7 +1882,7 @@ def test_generic_to_fit_with_hps(sagemaker_session): def test_generic_to_fit_with_experiment_config(sagemaker_session): e = Estimator( - IMAGE_NAME, + IMAGE_URI, ROLE, INSTANCE_COUNT, INSTANCE_TYPE, @@ -1902,7 +1902,7 @@ def test_generic_to_fit_with_experiment_config(sagemaker_session): sagemaker_session.train.assert_called_once() assert len(sagemaker_session.train.call_args[0]) == 0 args = sagemaker_session.train.call_args[1] - assert args["job_name"].startswith(IMAGE_NAME) + assert args["job_name"].startswith(IMAGE_URI) args.pop("job_name") args.pop("role") @@ -1912,7 +1912,7 @@ def test_generic_to_fit_with_experiment_config(sagemaker_session): def test_generic_to_fit_with_encrypt_inter_container_traffic_flag(sagemaker_session): e = Estimator( - IMAGE_NAME, + IMAGE_URI, ROLE, INSTANCE_COUNT, INSTANCE_TYPE, @@ -1930,7 +1930,7 @@ def test_generic_to_fit_with_encrypt_inter_container_traffic_flag(sagemaker_sess def test_generic_to_fit_with_network_isolation(sagemaker_session): e = Estimator( - IMAGE_NAME, + IMAGE_URI, ROLE, INSTANCE_COUNT, INSTANCE_TYPE, @@ -1948,7 +1948,7 @@ def test_generic_to_fit_with_network_isolation(sagemaker_session): def test_generic_to_fit_with_sagemaker_metrics_missing(sagemaker_session): e = Estimator( - IMAGE_NAME, + IMAGE_URI, ROLE, INSTANCE_COUNT, INSTANCE_TYPE, @@ -1965,7 +1965,7 @@ def test_generic_to_fit_with_sagemaker_metrics_missing(sagemaker_session): def test_generic_to_fit_with_sagemaker_metrics_enabled(sagemaker_session): e = Estimator( - IMAGE_NAME, + IMAGE_URI, ROLE, INSTANCE_COUNT, INSTANCE_TYPE, @@ -1983,7 +1983,7 @@ def test_generic_to_fit_with_sagemaker_metrics_enabled(sagemaker_session): def test_generic_to_fit_with_sagemaker_metrics_disabled(sagemaker_session): e = Estimator( - IMAGE_NAME, + IMAGE_URI, ROLE, INSTANCE_COUNT, INSTANCE_TYPE, @@ -2001,7 +2001,7 @@ def test_generic_to_fit_with_sagemaker_metrics_disabled(sagemaker_session): def test_generic_to_deploy(sagemaker_session): e = Estimator( - IMAGE_NAME, + IMAGE_URI, ROLE, INSTANCE_COUNT, INSTANCE_TYPE, @@ -2018,7 +2018,7 @@ def test_generic_to_deploy(sagemaker_session): sagemaker_session.train.assert_called_once() assert len(sagemaker_session.train.call_args[0]) == 0 args = sagemaker_session.train.call_args[1] - assert args["job_name"].startswith(IMAGE_NAME) + assert args["job_name"].startswith(IMAGE_URI) args.pop("job_name") args.pop("role") @@ -2027,20 +2027,20 @@ def test_generic_to_deploy(sagemaker_session): sagemaker_session.create_model.assert_called_once() args, kwargs = sagemaker_session.create_model.call_args - assert args[0].startswith(IMAGE_NAME) + assert args[0].startswith(IMAGE_URI) assert args[1] == ROLE - assert args[2]["Image"] == IMAGE_NAME + assert args[2]["Image"] == IMAGE_URI assert args[2]["ModelDataUrl"] == MODEL_DATA assert kwargs["vpc_config"] is None assert isinstance(predictor, Predictor) - assert predictor.endpoint_name.startswith(IMAGE_NAME) + assert predictor.endpoint_name.startswith(IMAGE_URI) assert predictor.sagemaker_session == sagemaker_session def test_generic_to_deploy_network_isolation(sagemaker_session): e = Estimator( - IMAGE_NAME, + IMAGE_URI, ROLE, INSTANCE_COUNT, INSTANCE_TYPE, @@ -2060,7 +2060,7 @@ def test_generic_to_deploy_network_isolation(sagemaker_session): @patch("sagemaker.estimator.Estimator.create_model") def test_generic_to_deploy_kms(create_model, sagemaker_session): e = Estimator( - IMAGE_NAME, + IMAGE_URI, ROLE, INSTANCE_COUNT, INSTANCE_TYPE, @@ -2106,7 +2106,7 @@ def test_generic_training_job_analytics(sagemaker_session): ) e = Estimator( - IMAGE_NAME, + IMAGE_URI, ROLE, INSTANCE_COUNT, INSTANCE_TYPE, @@ -2130,7 +2130,7 @@ def test_generic_create_model_vpc_config_override(sagemaker_session): vpc_config_b = {"Subnets": ["foo", "bar"], "SecurityGroupIds": ["baz"]} e = Estimator( - IMAGE_NAME, ROLE, INSTANCE_COUNT, INSTANCE_TYPE, sagemaker_session=sagemaker_session + IMAGE_URI, ROLE, INSTANCE_COUNT, INSTANCE_TYPE, sagemaker_session=sagemaker_session ) e.fit({"train": "s3://bucket/training-prefix"}) assert e.get_vpc_config() is None @@ -2156,7 +2156,7 @@ def test_generic_deploy_vpc_config_override(sagemaker_session): vpc_config_b = {"Subnets": ["foo", "bar"], "SecurityGroupIds": ["baz"]} e = Estimator( - IMAGE_NAME, ROLE, INSTANCE_COUNT, INSTANCE_TYPE, sagemaker_session=sagemaker_session + IMAGE_URI, ROLE, INSTANCE_COUNT, INSTANCE_TYPE, sagemaker_session=sagemaker_session ) e.fit({"train": "s3://bucket/training-prefix"}) e.deploy(INSTANCE_COUNT, INSTANCE_TYPE) @@ -2176,14 +2176,14 @@ def test_generic_deploy_vpc_config_override(sagemaker_session): def test_generic_deploy_accelerator_type(sagemaker_session): e = Estimator( - IMAGE_NAME, ROLE, INSTANCE_COUNT, INSTANCE_TYPE, sagemaker_session=sagemaker_session + IMAGE_URI, ROLE, INSTANCE_COUNT, INSTANCE_TYPE, sagemaker_session=sagemaker_session ) e.fit({"train": "s3://bucket/training-prefix"}) e.deploy(INSTANCE_COUNT, INSTANCE_TYPE, ACCELERATOR_TYPE) args = e.sagemaker_session.endpoint_from_production_variants.call_args[1] print(args) - assert args["name"].startswith(IMAGE_NAME) + assert args["name"].startswith(IMAGE_URI) assert args["production_variants"][0]["AcceleratorType"] == ACCELERATOR_TYPE assert args["production_variants"][0]["InitialInstanceCount"] == INSTANCE_COUNT assert args["production_variants"][0]["InstanceType"] == INSTANCE_TYPE @@ -2191,7 +2191,7 @@ def test_generic_deploy_accelerator_type(sagemaker_session): def test_deploy_with_model_name(sagemaker_session): estimator = Estimator( - IMAGE_NAME, + IMAGE_URI, ROLE, INSTANCE_COUNT, INSTANCE_TYPE, @@ -2210,7 +2210,7 @@ def test_deploy_with_model_name(sagemaker_session): def test_deploy_with_no_model_name(sagemaker_session): estimator = Estimator( - IMAGE_NAME, + IMAGE_URI, ROLE, INSTANCE_COUNT, INSTANCE_TYPE, @@ -2223,7 +2223,7 @@ def test_deploy_with_no_model_name(sagemaker_session): sagemaker_session.create_model.assert_called_once() args, kwargs = sagemaker_session.create_model.call_args - assert args[0].startswith(IMAGE_NAME) + assert args[0].startswith(IMAGE_URI) @patch("sagemaker.estimator.LocalSession") @@ -2238,21 +2238,21 @@ def test_local_mode(session_class, local_session_class): local_session_class.return_value = local_session session_class.return_value = session - e = Estimator(IMAGE_NAME, ROLE, INSTANCE_COUNT, "local") + e = Estimator(IMAGE_URI, ROLE, INSTANCE_COUNT, "local") print(e.sagemaker_session.local_mode) assert e.sagemaker_session.local_mode is True - e2 = Estimator(IMAGE_NAME, ROLE, INSTANCE_COUNT, "local_gpu") + e2 = Estimator(IMAGE_URI, ROLE, INSTANCE_COUNT, "local_gpu") assert e2.sagemaker_session.local_mode is True - e3 = Estimator(IMAGE_NAME, ROLE, INSTANCE_COUNT, INSTANCE_TYPE) + e3 = Estimator(IMAGE_URI, ROLE, INSTANCE_COUNT, INSTANCE_TYPE) assert e3.sagemaker_session.local_mode is False @patch("sagemaker.estimator.LocalSession") def test_distributed_gpu_local_mode(LocalSession): with pytest.raises(RuntimeError): - Estimator(IMAGE_NAME, ROLE, 3, "local_gpu", output_path=OUTPUT_PATH) + Estimator(IMAGE_URI, ROLE, 3, "local_gpu", output_path=OUTPUT_PATH) @patch("sagemaker.estimator.LocalSession") @@ -2261,7 +2261,7 @@ def test_local_mode_file_output_path(local_session_class): local_session.local_mode = True local_session_class.return_value = local_session - e = Estimator(IMAGE_NAME, ROLE, INSTANCE_COUNT, "local", output_path="file:///tmp/model/") + e = Estimator(IMAGE_URI, ROLE, INSTANCE_COUNT, "local", output_path="file:///tmp/model/") assert e.output_path == "file:///tmp/model/" @@ -2272,7 +2272,7 @@ def test_file_output_path_not_supported_outside_local_mode(session_class): session_class.return_value = session with pytest.raises(RuntimeError): - Estimator(IMAGE_NAME, ROLE, INSTANCE_COUNT, INSTANCE_TYPE, output_path="file:///tmp/model") + Estimator(IMAGE_URI, ROLE, INSTANCE_COUNT, INSTANCE_TYPE, output_path="file:///tmp/model") def test_prepare_init_params_from_job_description_with_image_training_job(): @@ -2318,7 +2318,7 @@ def test_prepare_init_params_from_job_description_with_invalid_training_job(): def test_prepare_for_training_with_base_name(sagemaker_session): estimator = Estimator( - image_name="some-image", + image_uri="some-image", role="some_image", train_instance_count=1, train_instance_type="ml.m4.xlarge", @@ -2332,7 +2332,7 @@ def test_prepare_for_training_with_base_name(sagemaker_session): def test_prepare_for_training_with_name_based_on_image(sagemaker_session): estimator = Estimator( - image_name="some-image", + image_uri="some-image", role="some_image", train_instance_count=1, train_instance_type="ml.m4.xlarge", @@ -2374,10 +2374,10 @@ def test_prepare_for_training_with_name_based_on_algorithm(sagemaker_session): ), ) def test_encryption_flag_in_non_vpc_mode_invalid(sagemaker_session): - image_name = registry("us-west-2") + "/factorization-machines:1" + image_uri = registry("us-west-2") + "/factorization-machines:1" with pytest.raises(ClientError) as error: estimator = Estimator( - image_name=image_name, + image_uri=image_uri, role="SageMakerRole", train_instance_count=1, train_instance_type="ml.c4.xlarge", @@ -2396,7 +2396,7 @@ def test_estimator_local_mode_error(sagemaker_session): # When using instance local with a session which is not LocalSession we should error out with pytest.raises(RuntimeError): Estimator( - image_name="some-image", + image_uri="some-image", role="some_image", train_instance_count=1, train_instance_type="local", @@ -2408,7 +2408,7 @@ def test_estimator_local_mode_error(sagemaker_session): def test_estimator_local_mode_ok(sagemaker_local_session): # When using instance local with a session which is not LocalSession we should error out Estimator( - image_name="some-image", + image_uri="some-image", role="some_image", train_instance_count=1, train_instance_type="local", diff --git a/tests/unit/test_fw_utils.py b/tests/unit/test_fw_utils.py index 8a3571355a..5270fc90cb 100644 --- a/tests/unit/test_fw_utils.py +++ b/tests/unit/test_fw_utils.py @@ -1117,47 +1117,47 @@ def walk(): def test_framework_name_from_image_mxnet(): - image_name = "123.dkr.ecr.us-west-2.amazonaws.com/sagemaker-mxnet:1.1-gpu-py3" - assert ("mxnet", "py3", "1.1-gpu-py3", None) == fw_utils.framework_name_from_image(image_name) + image_uri = "123.dkr.ecr.us-west-2.amazonaws.com/sagemaker-mxnet:1.1-gpu-py3" + assert ("mxnet", "py3", "1.1-gpu-py3", None) == fw_utils.framework_name_from_image(image_uri) def test_framework_name_from_image_mxnet_in_gov(): - image_name = "123.dkr.ecr.region-name.c2s.ic.gov/sagemaker-mxnet:1.1-gpu-py3" - assert ("mxnet", "py3", "1.1-gpu-py3", None) == fw_utils.framework_name_from_image(image_name) + image_uri = "123.dkr.ecr.region-name.c2s.ic.gov/sagemaker-mxnet:1.1-gpu-py3" + assert ("mxnet", "py3", "1.1-gpu-py3", None) == fw_utils.framework_name_from_image(image_uri) def test_framework_name_from_image_tf(): - image_name = "123.dkr.ecr.us-west-2.amazonaws.com/sagemaker-tensorflow:1.6-cpu-py2" + image_uri = "123.dkr.ecr.us-west-2.amazonaws.com/sagemaker-tensorflow:1.6-cpu-py2" assert ("tensorflow", "py2", "1.6-cpu-py2", None) == fw_utils.framework_name_from_image( - image_name + image_uri ) def test_framework_name_from_image_tf_scriptmode(): - image_name = "123.dkr.ecr.us-west-2.amazonaws.com/sagemaker-tensorflow-scriptmode:1.12-cpu-py3" + image_uri = "123.dkr.ecr.us-west-2.amazonaws.com/sagemaker-tensorflow-scriptmode:1.12-cpu-py3" assert ( "tensorflow", "py3", "1.12-cpu-py3", "scriptmode", - ) == fw_utils.framework_name_from_image(image_name) + ) == fw_utils.framework_name_from_image(image_uri) - image_name = "123.dkr.ecr.us-west-2.amazonaws.com/tensorflow-training:1.13-cpu-py3" + image_uri = "123.dkr.ecr.us-west-2.amazonaws.com/tensorflow-training:1.13-cpu-py3" assert ("tensorflow", "py3", "1.13-cpu-py3", "training") == fw_utils.framework_name_from_image( - image_name + image_uri ) def test_framework_name_from_image_rl(): - image_name = "123.dkr.ecr.us-west-2.amazonaws.com/sagemaker-rl-mxnet:toolkit1.1-gpu-py3" + image_uri = "123.dkr.ecr.us-west-2.amazonaws.com/sagemaker-rl-mxnet:toolkit1.1-gpu-py3" assert ("mxnet", "py3", "toolkit1.1-gpu-py3", None) == fw_utils.framework_name_from_image( - image_name + image_uri ) def test_legacy_name_from_framework_image(): - image_name = "123.dkr.ecr.us-west-2.amazonaws.com/sagemaker-mxnet-py3-gpu:2.5.6-gpu-py2" - framework, py_ver, tag, _ = fw_utils.framework_name_from_image(image_name) + image_uri = "123.dkr.ecr.us-west-2.amazonaws.com/sagemaker-mxnet-py3-gpu:2.5.6-gpu-py2" + framework, py_ver, tag, _ = fw_utils.framework_name_from_image(image_uri) assert framework == "mxnet" assert py_ver == "py3" assert tag == "2.5.6-gpu-py2" @@ -1191,8 +1191,8 @@ def test_legacy_name_from_wrong_device(): def test_legacy_name_from_image_any_tag(): - image_name = "123.dkr.ecr.us-west-2.amazonaws.com/sagemaker-tensorflow-py2-cpu:any-tag" - framework, py_ver, tag, _ = fw_utils.framework_name_from_image(image_name) + image_uri = "123.dkr.ecr.us-west-2.amazonaws.com/sagemaker-tensorflow-py2-cpu:any-tag" + framework, py_ver, tag, _ = fw_utils.framework_name_from_image(image_uri) assert framework == "tensorflow" assert py_ver == "py2" assert tag == "any-tag" @@ -1221,25 +1221,25 @@ def test_parse_s3_url_fail(): def test_model_code_key_prefix_with_all_values_present(): - key_prefix = fw_utils.model_code_key_prefix("prefix", "model_name", "image_name") + key_prefix = fw_utils.model_code_key_prefix("prefix", "model_name", "image_uri") assert key_prefix == "prefix/model_name" def test_model_code_key_prefix_with_no_prefix_and_all_other_values_present(): - key_prefix = fw_utils.model_code_key_prefix(None, "model_name", "image_name") + key_prefix = fw_utils.model_code_key_prefix(None, "model_name", "image_uri") assert key_prefix == "model_name" @patch("time.strftime", return_value=TIMESTAMP) def test_model_code_key_prefix_with_only_image_present(time): - key_prefix = fw_utils.model_code_key_prefix(None, None, "image_name") - assert key_prefix == name_from_image("image_name") + key_prefix = fw_utils.model_code_key_prefix(None, None, "image_uri") + assert key_prefix == name_from_image("image_uri") @patch("time.strftime", return_value=TIMESTAMP) def test_model_code_key_prefix_and_image_present(time): - key_prefix = fw_utils.model_code_key_prefix("prefix", None, "image_name") - assert key_prefix == "prefix/" + name_from_image("image_name") + key_prefix = fw_utils.model_code_key_prefix("prefix", None, "image_uri") + assert key_prefix == "prefix/" + name_from_image("image_uri") def test_model_code_key_prefix_with_prefix_present_and_others_none_fail(): @@ -1286,12 +1286,12 @@ def test_warn_if_parameter_server_with_local_multi_gpu(caplog): def test_validate_version_or_image_args_not_raises(): good_args = [("1.0", "py3", None), (None, "py3", "my:uri"), ("1.0", None, "my:uri")] - for framework_version, py_version, image_name in good_args: - fw_utils.validate_version_or_image_args(framework_version, py_version, image_name) + for framework_version, py_version, image_uri in good_args: + fw_utils.validate_version_or_image_args(framework_version, py_version, image_uri) def test_validate_version_or_image_args_raises(): bad_args = [(None, None, None), (None, "py3", None), ("1.0", None, None)] - for framework_version, py_version, image_name in bad_args: + for framework_version, py_version, image_uri in bad_args: with pytest.raises(ValueError): - fw_utils.validate_version_or_image_args(framework_version, py_version, image_name) + fw_utils.validate_version_or_image_args(framework_version, py_version, image_uri) diff --git a/tests/unit/test_mxnet.py b/tests/unit/test_mxnet.py index e6d68b44dd..12f92b0e6c 100644 --- a/tests/unit/test_mxnet.py +++ b/tests/unit/test_mxnet.py @@ -146,7 +146,7 @@ def _get_train_args(job_name): } -def _get_environment(submit_directory, model_url, image_name): +def _get_environment(submit_directory, model_url, image_uri): return { "Environment": { "SAGEMAKER_SUBMIT_DIRECTORY": submit_directory, @@ -155,7 +155,7 @@ def _get_environment(submit_directory, model_url, image_name): "SAGEMAKER_REGION": "us-west-2", "SAGEMAKER_CONTAINER_LOG_LEVEL": "20", }, - "Image": image_name, + "Image": image_uri, "ModelDataUrl": model_url, } @@ -277,7 +277,7 @@ def test_create_model_with_custom_image(name_from_base, sagemaker_session): sagemaker_session=sagemaker_session, train_instance_count=INSTANCE_COUNT, train_instance_type=INSTANCE_TYPE, - image_name=custom_image, + image_uri=custom_image, container_log_level=container_log_level, base_job_name=base_job_name, source_dir=source_dir, @@ -665,7 +665,7 @@ def test_attach_custom_image(sagemaker_session): ) estimator = MXNet.attach(training_job_name="neo", sagemaker_session=sagemaker_session) - assert estimator.image_name == training_image + assert estimator.image_uri == training_image assert estimator.train_image() == training_image @@ -741,14 +741,14 @@ def test_create_model_with_custom_hosting_image(sagemaker_session): sagemaker_session=sagemaker_session, train_instance_count=INSTANCE_COUNT, train_instance_type=INSTANCE_TYPE, - image_name=custom_image, + image_uri=custom_image, container_log_level=container_log_level, base_job_name="job", source_dir=source_dir, ) mx.fit(inputs="s3://mybucket/train", job_name="new_name") - model = mx.create_model(image_name=custom_hosting_image) + model = mx.create_model(image_uri=custom_hosting_image) assert model.image == custom_hosting_image diff --git a/tests/unit/test_pytorch.py b/tests/unit/test_pytorch.py index 26d68ac534..4057b01758 100644 --- a/tests/unit/test_pytorch.py +++ b/tests/unit/test_pytorch.py @@ -35,8 +35,8 @@ INSTANCE_COUNT = 1 INSTANCE_TYPE = "ml.c4.4xlarge" ACCELERATOR_TYPE = "ml.eia.medium" -IMAGE_NAME = "sagemaker-pytorch" -JOB_NAME = "{}-{}".format(IMAGE_NAME, TIMESTAMP) +IMAGE_URI = "sagemaker-pytorch" +JOB_NAME = "{}-{}".format(IMAGE_URI, TIMESTAMP) IMAGE_URI_FORMAT_STRING = "520713654638.dkr.ecr.{}.amazonaws.com/{}:{}-{}-{}" ROLE = "Dummy" REGION = "us-west-2" @@ -80,11 +80,11 @@ def fixture_sagemaker_session(): def _get_full_cpu_image_uri(version, py_version): - return IMAGE_URI_FORMAT_STRING.format(REGION, IMAGE_NAME, version, "cpu", py_version) + return IMAGE_URI_FORMAT_STRING.format(REGION, IMAGE_URI, version, "cpu", py_version) def _get_full_gpu_image_uri(version, py_version): - return IMAGE_URI_FORMAT_STRING.format(REGION, IMAGE_NAME, version, "gpu", py_version) + return IMAGE_URI_FORMAT_STRING.format(REGION, IMAGE_URI, version, "gpu", py_version) def _get_full_cpu_image_uri_with_ei(version, py_version): @@ -250,7 +250,7 @@ def test_create_model_with_custom_image(name_from_base, sagemaker_session): train_instance_count=INSTANCE_COUNT, train_instance_type=INSTANCE_TYPE, container_log_level=container_log_level, - image_name=image, + image_uri=image, base_job_name=base_job_name, source_dir=source_dir, ) @@ -566,7 +566,7 @@ def test_attach_custom_image(sagemaker_session): estimator = PyTorch.attach(training_job_name="neo", sagemaker_session=sagemaker_session) assert estimator.latest_training_job.job_name == "neo" - assert estimator.image_name == training_image + assert estimator.image_uri == training_image assert estimator.train_image() == training_image diff --git a/tests/unit/test_rl.py b/tests/unit/test_rl.py index bcd9124b2e..a956ea0ddb 100644 --- a/tests/unit/test_rl.py +++ b/tests/unit/test_rl.py @@ -32,7 +32,7 @@ BUCKET_NAME = "notmybucket" INSTANCE_COUNT = 1 INSTANCE_TYPE = "ml.c4.4xlarge" -IMAGE_NAME = "sagemaker-rl" +IMAGE_URI = "sagemaker-rl" IMAGE_URI_FORMAT_STRING = "520713654638.dkr.ecr.{}.amazonaws.com/{}-{}:{}{}-{}-py3" PYTHON_VERSION = "py3" ROLE = "Dummy" @@ -78,13 +78,13 @@ def fixture_sagemaker_session(): def _get_full_cpu_image_uri(toolkit, toolkit_version, framework): return IMAGE_URI_FORMAT_STRING.format( - REGION, IMAGE_NAME, framework, toolkit, toolkit_version, "cpu" + REGION, IMAGE_URI, framework, toolkit, toolkit_version, "cpu" ) def _get_full_gpu_image_uri(toolkit, toolkit_version, framework): return IMAGE_URI_FORMAT_STRING.format( - REGION, IMAGE_NAME, framework, toolkit, toolkit_version, "gpu" + REGION, IMAGE_URI, framework, toolkit, toolkit_version, "gpu" ) @@ -112,7 +112,7 @@ def _rl_estimator( def _create_train_job(toolkit, toolkit_version, framework): - job_name = "{}-{}-{}".format(IMAGE_NAME, framework, TIMESTAMP) + job_name = "{}-{}-{}".format(IMAGE_URI, framework, TIMESTAMP) return { "image": _get_full_cpu_image_uri(toolkit, toolkit_version, framework), "input_mode": "File", @@ -282,7 +282,7 @@ def test_create_model_with_custom_image(name_from_base, sagemaker_session): sagemaker_session=sagemaker_session, train_instance_count=INSTANCE_COUNT, train_instance_type=INSTANCE_TYPE, - image_name=image, + image_uri=image, container_log_level=container_log_level, source_dir=source_dir, ) @@ -574,7 +574,7 @@ def test_attach_custom_image(sagemaker_session): estimator = RLEstimator.attach(training_job_name="neo", sagemaker_session=sagemaker_session) assert estimator.latest_training_job.job_name == "neo" - assert estimator.image_name == training_image + assert estimator.image_uri == training_image assert estimator.train_image() == training_image @@ -622,7 +622,7 @@ def test_missing_required_parameters(sagemaker_session): train_instance_type=INSTANCE_TYPE, ) assert ( - "Please provide `toolkit`, `toolkit_version`, `framework`" + " or `image_name` parameter." + "Please provide `toolkit`, `toolkit_version`, `framework`" + " or `image_uri` parameter." in str(e.value) ) diff --git a/tests/unit/test_sklearn.py b/tests/unit/test_sklearn.py index 2c868fe406..845ac2e4bb 100644 --- a/tests/unit/test_sklearn.py +++ b/tests/unit/test_sklearn.py @@ -34,8 +34,8 @@ INSTANCE_TYPE = "ml.c4.4xlarge" GPU_INSTANCE_TYPE = "ml.p2.xlarge" PYTHON_VERSION = "py3" -IMAGE_NAME = "sagemaker-scikit-learn" -JOB_NAME = "{}-{}".format(IMAGE_NAME, TIMESTAMP) +IMAGE_URI = "sagemaker-scikit-learn" +JOB_NAME = "{}-{}".format(IMAGE_URI, TIMESTAMP) IMAGE_URI_FORMAT_STRING = "246618743249.dkr.ecr.{}.amazonaws.com/{}:{}-{}-{}" ROLE = "Dummy" REGION = "us-west-2" @@ -78,7 +78,7 @@ def sagemaker_session(): def _get_full_cpu_image_uri(version): - return IMAGE_URI_FORMAT_STRING.format(REGION, IMAGE_NAME, version, "cpu", PYTHON_VERSION) + return IMAGE_URI_FORMAT_STRING.format(REGION, IMAGE_URI, version, "cpu", PYTHON_VERSION) def _sklearn_estimator( @@ -293,7 +293,7 @@ def test_create_model_with_custom_image(sagemaker_session): role=ROLE, sagemaker_session=sagemaker_session, train_instance_type=INSTANCE_TYPE, - image_name=custom_image, + image_uri=custom_image, container_log_level=container_log_level, py_version=PYTHON_VERSION, base_job_name="job", @@ -563,7 +563,7 @@ def test_attach_custom_image(sagemaker_session): ) estimator = SKLearn.attach(training_job_name="neo", sagemaker_session=sagemaker_session) - assert estimator.image_name == training_image + assert estimator.image_uri == training_image assert estimator.train_image() == training_image diff --git a/tests/unit/test_transformer.py b/tests/unit/test_transformer.py index aeaa290827..a59ab6499b 100644 --- a/tests/unit/test_transformer.py +++ b/tests/unit/test_transformer.py @@ -19,7 +19,7 @@ from tests.integ import test_local_mode MODEL_NAME = "model" -IMAGE_NAME = "image-for-model" +IMAGE_URI = "image-for-model" JOB_NAME = "job" INSTANCE_COUNT = 1 @@ -40,9 +40,9 @@ "base_transform_job_name": JOB_NAME, } -MODEL_DESC_PRIMARY_CONTAINER = {"PrimaryContainer": {"Image": IMAGE_NAME}} +MODEL_DESC_PRIMARY_CONTAINER = {"PrimaryContainer": {"Image": IMAGE_URI}} -MODEL_DESC_CONTAINERS_ONLY = {"Containers": [{"Image": IMAGE_NAME}]} +MODEL_DESC_CONTAINERS_ONLY = {"Containers": [{"Image": IMAGE_URI}]} @pytest.fixture(autouse=True) @@ -212,33 +212,33 @@ def test_transform_with_base_job_name_provided(start_new_job, name_from_base, tr assert transformer._current_job_name == full_name -@patch("sagemaker.transformer.Transformer._retrieve_base_name", return_value=IMAGE_NAME) +@patch("sagemaker.transformer.Transformer._retrieve_base_name", return_value=IMAGE_URI) @patch("sagemaker.transformer.name_from_base") @patch("sagemaker.transformer._TransformJob.start_new") def test_transform_with_base_name(start_new_job, name_from_base, retrieve_base_name, transformer): - full_name = "{}-{}".format(IMAGE_NAME, TIMESTAMP) + full_name = "{}-{}".format(IMAGE_URI, TIMESTAMP) name_from_base.return_value = full_name transformer.transform(DATA) retrieve_base_name.assert_called_once_with() - name_from_base.assert_called_once_with(IMAGE_NAME) + name_from_base.assert_called_once_with(IMAGE_URI) assert transformer._current_job_name == full_name -@patch("sagemaker.transformer.Transformer._retrieve_image_name", return_value=IMAGE_NAME) +@patch("sagemaker.transformer.Transformer._retrieve_image_uri", return_value=IMAGE_URI) @patch("sagemaker.transformer.name_from_base") @patch("sagemaker.transformer._TransformJob.start_new") def test_transform_with_job_name_based_on_image( - start_new_job, name_from_base, retrieve_image_name, transformer + start_new_job, name_from_base, retrieve_image_uri, transformer ): - full_name = "{}-{}".format(IMAGE_NAME, TIMESTAMP) + full_name = "{}-{}".format(IMAGE_URI, TIMESTAMP) name_from_base.return_value = full_name transformer.transform(DATA) - retrieve_image_name.assert_called_once_with() - name_from_base.assert_called_once_with(IMAGE_NAME) + retrieve_image_uri.assert_called_once_with() + name_from_base.assert_called_once_with(IMAGE_URI) assert transformer._current_job_name == full_name @@ -250,7 +250,7 @@ def test_transform_with_job_name_based_on_containers( ): transformer.sagemaker_session.sagemaker_client.describe_model.return_value = model_desc - full_name = "{}-{}".format(IMAGE_NAME, TIMESTAMP) + full_name = "{}-{}".format(IMAGE_URI, TIMESTAMP) name_from_base.return_value = full_name transformer.transform(DATA) @@ -258,7 +258,7 @@ def test_transform_with_job_name_based_on_containers( transformer.sagemaker_session.sagemaker_client.describe_model.assert_called_once_with( ModelName=MODEL_NAME ) - name_from_base.assert_called_once_with(IMAGE_NAME) + name_from_base.assert_called_once_with(IMAGE_URI) assert transformer._current_job_name == full_name @@ -300,13 +300,13 @@ def test_transform_with_invalid_s3_uri(transformer): assert "Invalid S3 URI" in str(e) -def test_retrieve_image_name(sagemaker_session, transformer): +def test_retrieve_image_uri(sagemaker_session, transformer): sage_mock = Mock(name="sagemaker_client") - sage_mock.describe_model.return_value = {"PrimaryContainer": {"Image": IMAGE_NAME}} + sage_mock.describe_model.return_value = {"PrimaryContainer": {"Image": IMAGE_URI}} sagemaker_session.sagemaker_client = sage_mock - assert transformer._retrieve_image_name() == IMAGE_NAME + assert transformer._retrieve_image_uri() == IMAGE_URI @patch("sagemaker.transformer.Transformer._ensure_last_transform_job") diff --git a/tests/unit/test_xgboost.py b/tests/unit/test_xgboost.py index f99c14018a..85b2c2dedb 100644 --- a/tests/unit/test_xgboost.py +++ b/tests/unit/test_xgboost.py @@ -35,8 +35,8 @@ INSTANCE_TYPE = "ml.c4.4xlarge" GPU_INSTANCE_TYPE = "ml.p2.xlarge" PYTHON_VERSION = "py3" -IMAGE_NAME = "sagemaker-xgboost" -JOB_NAME = "{}-{}".format(IMAGE_NAME, TIMESTAMP) +IMAGE_URI = "sagemaker-xgboost" +JOB_NAME = "{}-{}".format(IMAGE_URI, TIMESTAMP) IMAGE_URI_FORMAT_STRING = "246618743249.dkr.ecr.{}.amazonaws.com/{}:{}-{}-{}" ROLE = "Dummy" REGION = "us-west-2" @@ -79,7 +79,7 @@ def sagemaker_session(): def _get_full_cpu_image_uri(version): - return IMAGE_URI_FORMAT_STRING.format(REGION, IMAGE_NAME, version, "cpu", PYTHON_VERSION) + return IMAGE_URI_FORMAT_STRING.format(REGION, IMAGE_URI, version, "cpu", PYTHON_VERSION) def _xgboost_estimator( @@ -284,7 +284,7 @@ def test_create_model_with_custom_image(sagemaker_session, xgboost_full_version) sagemaker_session=sagemaker_session, train_instance_type=INSTANCE_TYPE, train_instance_count=1, - image_name=custom_image, + image_uri=custom_image, container_log_level=container_log_level, py_version=PYTHON_VERSION, base_job_name="job",