breaking: deprecated constants from defaults

Author: metrizable

src/sagemaker/amazon/amazon_estimator.py

@@ -611,6 +611,8 @@ def get_image_uri(region_name, repo_name, repo_version=1):
     """Return algorithm image URI for the given AWS region, repository name, and
     repository version
 
+    TODO: consider refactoring version/repo validation into a module for all frameworks
+
     Args:
         region_name:
         repo_name:

src/sagemaker/chainer/defaults.py

@@ -13,12 +13,4 @@
 """Placeholder docstring"""
 from __future__ import absolute_import
 
-CHAINER_VERSION = "4.1.0"
-"""Default Chainer version for when the framework version is not specified.
-This is no longer updated so as to not break existing workflows.
-"""
-
-LATEST_VERSION = "5.0.0"
-"""The latest version of Chainer included in the SageMaker pre-built Docker images."""
-
 LATEST_PY2_VERSION = "5.0.0"

src/sagemaker/chainer/estimator.py

@@ -19,7 +19,6 @@
 from sagemaker.fw_utils import (
     framework_name_from_image,
     framework_version_from_tag,
-    empty_framework_version_warning,
     python_deprecation_warning,
 )
 from sagemaker.chainer import defaults
@@ -40,8 +39,6 @@ class Chainer(Framework):
     _process_slots_per_host = "sagemaker_process_slots_per_host"
     _additional_mpi_options = "sagemaker_additional_mpi_options"
 
-    LATEST_VERSION = defaults.LATEST_VERSION
-
     def __init__(
         self,
         entry_point,
@@ -126,11 +123,7 @@ def __init__(
             :class:`~sagemaker.estimator.Framework` and
             :class:`~sagemaker.estimator.EstimatorBase`.
         """
-        if framework_version is None:
-            logger.warning(
-                empty_framework_version_warning(defaults.CHAINER_VERSION, self.LATEST_VERSION)
-            )
-        self.framework_version = framework_version or defaults.CHAINER_VERSION
+        self.framework_version = framework_version
 
         super(Chainer, self).__init__(
             entry_point, source_dir, hyperparameters, image_name=image_name, **kwargs

src/sagemaker/chainer/model.py

@@ -18,12 +18,7 @@
 from sagemaker import fw_utils
 
 import sagemaker
-from sagemaker.fw_utils import (
-    create_image_uri,
-    model_code_key_prefix,
-    python_deprecation_warning,
-    empty_framework_version_warning,
-)
+from sagemaker.fw_utils import create_image_uri, model_code_key_prefix, python_deprecation_warning
 from sagemaker.model import FrameworkModel, MODEL_SERVER_WORKERS_PARAM_NAME
 from sagemaker.chainer import defaults
 from sagemaker.predictor import RealTimePredictor, npy_serializer, numpy_deserializer
@@ -117,13 +112,8 @@ def __init__(
                 python_deprecation_warning(self.__framework_name__, defaults.LATEST_PY2_VERSION)
             )
 
-        if framework_version is None:
-            logger.warning(
-                empty_framework_version_warning(defaults.CHAINER_VERSION, defaults.LATEST_VERSION)
-            )
-
         self.py_version = py_version
-        self.framework_version = framework_version or defaults.CHAINER_VERSION
+        self.framework_version = framework_version
         self.model_server_workers = model_server_workers
 
     def prepare_container_def(self, instance_type, accelerator_type=None):

src/sagemaker/cli/mxnet.py

@@ -14,7 +14,8 @@
 from __future__ import absolute_import
 
 from sagemaker.cli.common import HostCommand, TrainCommand
-from sagemaker.mxnet import defaults
+
+MXNET_VERSION = "1.2"
 
 
 def train(args):
@@ -42,7 +43,7 @@ def create_estimator(self):
 
         return MXNet(
             entry_point=self.script,
-            framework_version=defaults.MXNET_VERSION,
+            framework_version=MXNET_VERSION,
             py_version=self.python,
             role=self.role_name,
             base_job_name=self.job_name,
@@ -66,7 +67,7 @@ def create_model(self, model_url):
             model_data=model_url,
             role=self.role_name,
             entry_point=self.script,
-            framework_version=defaults.MXNET_VERSION,
+            framework_version=MXNET_VERSION,
             py_version=self.python,
             name=self.endpoint_name,
             env=self.environment,

src/sagemaker/mxnet/defaults.py

@@ -13,12 +13,4 @@
 """Placeholder docstring"""
 from __future__ import absolute_import
 
-MXNET_VERSION = "1.2"
-"""Default MXNet version for when the framework version is not specified.
-This is no longer updated so as to not break existing workflows.
-"""
-
-LATEST_VERSION = "1.6.0"
-"""The latest version of MXNet included in the SageMaker pre-built Docker images."""
-
 LATEST_PY2_VERSION = "1.6.0"

src/sagemaker/mxnet/estimator.py

@@ -38,8 +38,6 @@ class MXNet(Framework):
     __framework_name__ = "mxnet"
     _LOWEST_SCRIPT_MODE_VERSION = ["1", "3"]
 
-    LATEST_VERSION = defaults.LATEST_VERSION
-
     def __init__(
         self,
         entry_point,
@@ -115,7 +113,7 @@ def __init__(
             :class:`~sagemaker.estimator.EstimatorBase`.
         """
         validate_version_or_image_args(framework_version, py_version, image_name)
-        if py_version and py_version == "py2":
+        if py_version == "py2":
             logger.warning(
                 python_deprecation_warning(self.__framework_name__, defaults.LATEST_PY2_VERSION)
             )

src/sagemaker/mxnet/model.py

@@ -115,7 +115,7 @@ def __init__(
             :class:`~sagemaker.model.Model`.
         """
         validate_version_or_image_args(framework_version, py_version, image)
-        if py_version and py_version == "py2":
+        if py_version == "py2":
             logger.warning(
                 python_deprecation_warning(self.__framework_name__, defaults.LATEST_PY2_VERSION)
             )

src/sagemaker/pytorch/defaults.py

@@ -13,14 +13,4 @@
 """Placeholder docstring"""
 from __future__ import absolute_import
 
-PYTORCH_VERSION = "0.4"
-"""Default PyTorch version for when the framework version is not specified.
-The default version is no longer updated so as to not break existing workflows.
-"""
-
-LATEST_VERSION = "1.5.0"
-"""The latest version of PyTorch included in the SageMaker pre-built Docker images."""
-
-PYTHON_VERSION = "py3"
-
 LATEST_PY2_VERSION = "1.3.1"

src/sagemaker/pytorch/estimator.py

@@ -35,8 +35,6 @@ class PyTorch(Framework):
 
     __framework_name__ = "pytorch"
 
-    LATEST_VERSION = defaults.LATEST_VERSION
-
     def __init__(
         self,
         entry_point,

src/sagemaker/pytorch/model.py

@@ -114,7 +114,7 @@ def __init__(
             :class:`~sagemaker.model.Model`.
         """
         validate_version_or_image_args(framework_version, py_version, image)
-        if py_version and py_version == "py2":
+        if py_version == "py2":
             logger.warning(
                 python_deprecation_warning(self.__framework_name__, defaults.LATEST_PY2_VERSION)
             )

src/sagemaker/sklearn/defaults.py

@@ -14,7 +14,3 @@
 from __future__ import absolute_import
 
 SKLEARN_NAME = "scikit-learn"
-
-SKLEARN_VERSION = "0.20.0"
-
-LATEST_PY2_VERSION = "0.20.0"

src/sagemaker/tensorflow/defaults.py

@@ -13,15 +13,4 @@
 """Placeholder docstring"""
 from __future__ import absolute_import
 
-TF_VERSION = "1.11"
-"""Default TF version for when the framework version is not specified.
-This is no longer updated so as to not break existing workflows.
-"""
-
-LATEST_VERSION = "2.2.0"
-"""The latest version of TensorFlow included in the SageMaker pre-built Docker images."""
-
-LATEST_SERVING_VERSION = "2.1.0"
-"""The latest version of TensorFlow Serving included in the SageMaker pre-built Docker images."""
-
 LATEST_PY2_VERSION = "2.1.0"

src/sagemaker/tensorflow/estimator.py

@@ -36,10 +36,6 @@ class TensorFlow(Framework):
     __framework_name__ = "tensorflow"
     _ECR_REPO_NAME = "tensorflow-scriptmode"
 
-    LATEST_VERSION = defaults.LATEST_VERSION
-
-    _LATEST_1X_VERSION = "1.15.2"
-
     _HIGHEST_LEGACY_MODE_ONLY_VERSION = version.Version("1.10.0")
     _HIGHEST_PYTHON_2_VERSION = version.Version("2.1.0")
 

src/sagemaker/xgboost/defaults.py

@@ -15,14 +15,20 @@
 
 XGBOOST_NAME = "xgboost"
 XGBOOST_1P_VERSIONS = ["1", "latest"]
-XGBOOST_VERSION_0_90 = "0.90"
+
+# TODO: evaluate usefulness of these constants. they are defined here and only used here
 XGBOOST_VERSION_0_90_1 = "0.90-1"
 XGBOOST_VERSION_0_90_2 = "0.90-2"
+
 XGBOOST_LATEST_VERSION = "1.0-1"
+
 # XGBOOST_SUPPORTED_VERSIONS has XGBoost Framework versions sorted from oldest to latest
 XGBOOST_SUPPORTED_VERSIONS = [
     XGBOOST_VERSION_0_90_1,
     XGBOOST_VERSION_0_90_2,
     XGBOOST_LATEST_VERSION,
 ]
+
+# TODO: evaluate use of this constant. it's used in precisely one place in different a module
+# may possibly be unnecessary indirection
 XGBOOST_VERSION_EQUIVALENTS = ["-cpu-py3"]

tests/conftest.py

@@ -21,14 +21,8 @@
 from botocore.config import Config
 
 from sagemaker import Session, utils
-from sagemaker.chainer import Chainer
 from sagemaker.local import LocalSession
-from sagemaker.mxnet import MXNet
-from sagemaker.pytorch import PyTorch
 from sagemaker.rl import RLEstimator
-from sagemaker.sklearn.defaults import SKLEARN_VERSION
-from sagemaker.tensorflow import TensorFlow
-from sagemaker.tensorflow.defaults import LATEST_VERSION, LATEST_SERVING_VERSION
 
 DEFAULT_REGION = "us-west-2"
 CUSTOM_BUCKET_NAME_PREFIX = "sagemaker-custom-bucket"
@@ -44,15 +38,19 @@
 
 NO_T2_REGIONS = ["eu-north-1", "ap-east-1", "me-south-1"]
 
+# TODO: refactor handling of versions, repo, image uris, validations for all frameworks
+TENSORFLOW_LATEST_VERSION = "2.2.0"
+TENSORFLOW_LATEST_1X_VERSION = "1.15.2"
+
 
 def pytest_addoption(parser):
     parser.addoption("--sagemaker-client-config", action="store", default=None)
     parser.addoption("--sagemaker-runtime-config", action="store", default=None)
     parser.addoption("--boto-config", action="store", default=None)
-    parser.addoption("--chainer-full-version", action="store", default=Chainer.LATEST_VERSION)
-    parser.addoption("--mxnet-full-version", action="store", default=MXNet.LATEST_VERSION)
+    parser.addoption("--chainer-full-version", action="store", default="5.0.0")
+    parser.addoption("--mxnet-full-version", action="store", default="1.6.0")
     parser.addoption("--ei-mxnet-full-version", action="store", default="1.5.1")
-    parser.addoption("--pytorch-full-version", action="store", default=PyTorch.LATEST_VERSION)
+    parser.addoption("--pytorch-full-version", action="store", default="1.5.0")
     parser.addoption(
         "--rl-coach-mxnet-full-version",
         action="store",
@@ -64,10 +62,10 @@ def pytest_addoption(parser):
     parser.addoption(
         "--rl-ray-full-version", action="store", default=RLEstimator.RAY_LATEST_VERSION
     )
-    parser.addoption("--sklearn-full-version", action="store", default=SKLEARN_VERSION)
+    parser.addoption("--sklearn-full-version", action="store", default="0.20.0")
     parser.addoption("--tf-full-version", action="store")
     parser.addoption("--ei-tf-full-version", action="store")
-    parser.addoption("--xgboost-full-version", action="store", default=SKLEARN_VERSION)
+    parser.addoption("--xgboost-full-version", action="store", default="1.0-1")
 
 
 def pytest_configure(config):
@@ -291,7 +289,27 @@ def sklearn_full_version(request):
     return request.config.getoption("--sklearn-full-version")
 
 
-@pytest.fixture(scope="module", params=[TensorFlow._LATEST_1X_VERSION, LATEST_VERSION])
+@pytest.fixture(scope="module", params=[TENSORFLOW_LATEST_VERSION])
+def tf_latest_version(request):
+    return request.param
+
+
+@pytest.fixture(scope="module")
+def tf_latest_py_version():
+    return "py37"
+
+
+@pytest.fixture(scope="module", params=[TENSORFLOW_LATEST_1X_VERSION])
+def tf_latest_1x_version(request):
+    return request.param
+
+
+@pytest.fixture(scope="module")
+def tf_latest_serving_version():
+    return "2.1.0"
+
+
+@pytest.fixture(scope="module", params=[TENSORFLOW_LATEST_VERSION, TENSORFLOW_LATEST_1X_VERSION])
 def tf_full_version(request):
     tf_version = request.config.getoption("--tf-full-version")
     if tf_version is None:
@@ -301,7 +319,7 @@ def tf_full_version(request):
 
 
 @pytest.fixture(scope="module")
-def tf_full_py_version(tf_full_version, request):
+def tf_full_py_version(tf_full_version, tf_latest_version, tf_latest_1x_version):
     """fixture to match tf_full_version
 
     Fixture exists as such, since tf_full_version may be overridden --tf-full-version.
@@ -312,11 +330,18 @@ def tf_full_py_version(tf_full_version, request):
     version = [int(val) for val in tf_full_version.split(".")]
     if version < [1, 11]:
         return "py2"
-    if tf_full_version in [TensorFlow._LATEST_1X_VERSION, LATEST_VERSION]:
+    if tf_full_version in [tf_latest_version, tf_latest_1x_version]:
         return "py37"
     return "py3"
 
 
+@pytest.fixture(scope="module")
+def tf_serving_version(tf_full_version, tf_latest_version, tf_latest_serving_version):
+    if tf_full_version == tf_latest_version:
+        return tf_latest_serving_version
+    return tf_full_version
+
+
 @pytest.fixture(scope="module", params=["1.15.0", "2.0.0"])
 def ei_tf_full_version(request):
     tf_ei_version = request.config.getoption("--ei-tf-full-version")
@@ -384,10 +409,3 @@ def pytest_generate_tests(metafunc):
 @pytest.fixture(scope="module")
 def xgboost_full_version(request):
     return request.config.getoption("--xgboost-full-version")
-
-
-@pytest.fixture(scope="module")
-def tf_serving_version(tf_full_version):
-    if tf_full_version == LATEST_VERSION:
-        return LATEST_SERVING_VERSION
-    return tf_full_version