From 8c8092cd9ff47c13dbac044449f31917d3c7b35f Mon Sep 17 00:00:00 2001 From: Miyoung Choi Date: Mon, 8 Aug 2022 13:27:46 -0700 Subject: [PATCH 1/4] add more info about volume_size --- src/sagemaker/estimator.py | 80 ++++++++++++++++++++++++++++++++++---- 1 file changed, 72 insertions(+), 8 deletions(-) diff --git a/src/sagemaker/estimator.py b/src/sagemaker/estimator.py index 9d0c30ff27..84c9e5499d 100644 --- a/src/sagemaker/estimator.py +++ b/src/sagemaker/estimator.py @@ -165,10 +165,42 @@ def __init__( instance_type (str): Type of EC2 instance to use for training, for example, ``'ml.c4.xlarge'``. Required if instance_groups is not set. - volume_size (int): Size in GB of the EBS volume to use for - storing input data during training (default: 30). Must be large - enough to store training data if File Mode is used (which is the - default). + volume_size (int): Size in GB of the storage volume to use for + storing input and output data during training (default: 30). + + Must be large enough to store training data if File mode is + used, which is the default mode. + + When using an ML instance with the EBS-only storage option and + without instance storage such as `NVMe SSD volumes + `_, + you must define the size of EBS + volume through the ``volume_size`` parameter in the estimator class. + For example, ML instance families that use EBS volumes include + ``ml.c5`` and ``ml.p2``. + + .. note:: + + When using an ML instance with `NVMe SSD volumes + `_, + SageMaker doesn't provision Amazon EBS General Purpose SSD + (gp2) storage. + Available storage is fixed to the NVMe-type instance's storage + capacity. SageMaker configures storage paths for training + datasets, checkpoints, model artifacts, and outputs to use the + entire capacity of the instance storage. For example, ML + instance families with the NVMe-type instance storage include + ``ml.p4d``, ``ml.g4dn``, and ``ml.g5``. + + To look up instance types and their instance storage types + and volumes, see `Amazon EC2 Instance Types + `_. + + To find the default local paths defined by the SageMaker + training platform, see `Amazon SageMaker Training Storage + Folders for Training Datasets, Checkpoints, Model Artifacts, + and Outputs + `_. volume_kms_key (str): Optional. KMS key ID for encrypting EBS volume attached to the training instance (default: None). max_run (int): Timeout in seconds for training (default: 24 * @@ -2196,10 +2228,42 @@ def __init__( instance_type (str): Type of EC2 instance to use for training, for example, 'ml.c4.xlarge'. Required if instance_groups is not set. - volume_size (int): Size in GB of the EBS volume to use for - storing input data during training (default: 30). Must be large - enough to store training data if File Mode is used (which is the - default). + volume_size (int): Size in GB of the storage volume to use for + storing input and output data during training (default: 30). + + Must be large enough to store training data if File mode is + used, which is the default mode. + + When using an ML instance with the EBS-only storage option and + without instance storage such as `NVMe SSD volumes + `_, + you must define the size of EBS + volume through the ``volume_size`` parameter in the estimator class. + For example, ML instance families that use EBS volumes include + ``ml.c5`` and ``ml.p2``. + + .. note:: + + When using an ML instance with `NVMe SSD volumes + `_, + SageMaker doesn't provision Amazon EBS General Purpose SSD + (gp2) storage. + Available storage is fixed to the NVMe-type instance's storage + capacity. SageMaker configures storage paths for training + datasets, checkpoints, model artifacts, and outputs to use the + entire capacity of the instance storage. For example, ML + instance families with the NVMe-type instance storage include + ``ml.p4d``, ``ml.g4dn``, and ``ml.g5``. + + To look up instance types and their instance storage types + and volumes, see `Amazon EC2 Instance Types + `_. + + To find the default local paths defined by the SageMaker + training platform, see `Amazon SageMaker Training Storage + Folders for Training Datasets, Checkpoints, Model Artifacts, + and Outputs + `_. volume_kms_key (str): Optional. KMS key ID for encrypting EBS volume attached to the training instance (default: None). max_run (int): Timeout in seconds for training (default: 24 * From 23402228834477b9759480cb4131c7d83dca0427 Mon Sep 17 00:00:00 2001 From: Miyoung Choi Date: Tue, 9 Aug 2022 13:59:19 -0700 Subject: [PATCH 2/4] Trigger Build From e20359f2b1c86828c511a9b813b558972c3939c6 Mon Sep 17 00:00:00 2001 From: Miyoung Choi Date: Mon, 29 Aug 2022 12:52:45 -0700 Subject: [PATCH 3/4] rephrase the note --- src/sagemaker/estimator.py | 40 ++++++++++++++++++++++++-------------- 1 file changed, 25 insertions(+), 15 deletions(-) diff --git a/src/sagemaker/estimator.py b/src/sagemaker/estimator.py index 4e241e7491..6fabe33a4f 100644 --- a/src/sagemaker/estimator.py +++ b/src/sagemaker/estimator.py @@ -182,16 +182,21 @@ def __init__( .. note:: - When using an ML instance with `NVMe SSD volumes - `_, + When you use an ML instance with `NVMe SSD volumes + `_ + such as ``ml.p4d``, ``ml.g4dn``, and ``ml.g5``, + do not include this parameter in the estimator configuration. + If you use one of those ML instance types, SageMaker doesn't provision Amazon EBS General Purpose SSD - (gp2) storage. - Available storage is fixed to the NVMe-type instance's storage + (gp2) storage nor take this parameter to adjust the NVMe instance storage. + Available storage is fixed to the NVMe instance storage capacity. SageMaker configures storage paths for training datasets, checkpoints, model artifacts, and outputs to use the - entire capacity of the instance storage. For example, ML - instance families with the NVMe-type instance storage include - ``ml.p4d``, ``ml.g4dn``, and ``ml.g5``. + entire capacity of the instance storage. + + Note that if you include this parameter and specify a number that + exceeds the size of the NVMe volume attached to the instance type, + SageMaker returns an ``Invalid VolumeSizeInGB`` error. To look up instance types and their instance storage types and volumes, see `Amazon EC2 Instance Types @@ -2264,7 +2269,7 @@ def __init__( instance_count (int): Number of Amazon EC2 instances to use for training. Required if instance_groups is not set. instance_type (str): Type of EC2 instance to use for training, - for example, 'ml.c4.xlarge'. Required if instance_groups is + for example, ``'ml.c4.xlarge'``. Required if instance_groups is not set. volume_size (int): Size in GB of the storage volume to use for storing input and output data during training (default: 30). @@ -2282,16 +2287,21 @@ def __init__( .. note:: - When using an ML instance with `NVMe SSD volumes - `_, + When you use an ML instance with `NVMe SSD volumes + `_ + such as ``ml.p4d``, ``ml.g4dn``, and ``ml.g5``, + do not include this parameter in the estimator configuration. + If you use one of those ML instance types, SageMaker doesn't provision Amazon EBS General Purpose SSD - (gp2) storage. - Available storage is fixed to the NVMe-type instance's storage + (gp2) storage nor take this parameter to adjust the NVMe instance storage. + Available storage is fixed to the NVMe instance storage capacity. SageMaker configures storage paths for training datasets, checkpoints, model artifacts, and outputs to use the - entire capacity of the instance storage. For example, ML - instance families with the NVMe-type instance storage include - ``ml.p4d``, ``ml.g4dn``, and ``ml.g5``. + entire capacity of the instance storage. + + Note that if you include this parameter and specify a number that + exceeds the size of the NVMe volume attached to the instance type, + SageMaker returns an ``Invalid VolumeSizeInGB`` error. To look up instance types and their instance storage types and volumes, see `Amazon EC2 Instance Types From f73d6bafe239d8c158a7fc78ded16cdab88c658e Mon Sep 17 00:00:00 2001 From: Miyoung Choi Date: Mon, 29 Aug 2022 14:36:24 -0700 Subject: [PATCH 4/4] minor rephrasing --- src/sagemaker/estimator.py | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/src/sagemaker/estimator.py b/src/sagemaker/estimator.py index 6fabe33a4f..4cb03c466e 100644 --- a/src/sagemaker/estimator.py +++ b/src/sagemaker/estimator.py @@ -172,13 +172,10 @@ def __init__( Must be large enough to store training data if File mode is used, which is the default mode. - When using an ML instance with the EBS-only storage option and - without instance storage such as `NVMe SSD volumes - `_, - you must define the size of EBS + When you use an ML instance with the EBS-only storage option + such as ``ml.c5`` and ``ml.p2``, + you must define the size of the EBS volume through the ``volume_size`` parameter in the estimator class. - For example, ML instance families that use EBS volumes include - ``ml.c5`` and ``ml.p2``. .. note:: @@ -2277,13 +2274,10 @@ def __init__( Must be large enough to store training data if File mode is used, which is the default mode. - When using an ML instance with the EBS-only storage option and - without instance storage such as `NVMe SSD volumes - `_, - you must define the size of EBS + When you use an ML instance with the EBS-only storage option + such as ``ml.c5`` and ``ml.p2``, + you must define the size of the EBS volume through the ``volume_size`` parameter in the estimator class. - For example, ML instance families that use EBS volumes include - ``ml.c5`` and ``ml.p2``. .. note::