change: include py38 tox env and some dependency upgrades (#1593)

Author: metrizable

.pylintrc

@@ -76,21 +76,22 @@ confidence=
 # --disable=W"
 disable=
     C0330, # Black disagrees with and explicitly violates this: https://github.com/python/black/issues/48
-    too-many-locals,
+    abstract-method, # TODO: Fix abstract methods
     arguments-differ,
-    too-many-lines,
+    cyclic-import, # TODO: Resolve cyclic imports
     fixme,
-    too-many-arguments,
     invalid-name,
-    too-many-instance-attributes,
-    len-as-condition, # TODO: Enable this check once pylint 2.4.0 is released and consumed due to the fix in https://github.com/PyCQA/pylint/issues/2684
     import-error, # Since we run Pylint before any of our builds in tox, this will always fail
-    protected-access, # TODO: Fix access
-    abstract-method, # TODO: Fix abstract methods
-    useless-object-inheritance, # TODO: Enable this check and fix code once Python 2 is no longer supported.
-    cyclic-import, # TODO: Resolve cyclic imports
+    import-outside-toplevel,
     no-self-use, # TODO: Convert methods to functions where appropriate
+    protected-access, # TODO: Fix access
+    signature-differs,  # TODO: fix kwargs
+    too-many-arguments,
     too-many-branches, # TODO: Simplify or ignore as appropriate
+    too-many-instance-attributes,
+    too-many-lines,
+    too-many-locals,
+    useless-object-inheritance, # TODO: Enable this check and fix code once Python 2 is no longer supported.
 
 [REPORTS]
 # Set the output format. Available formats are text, parseable, colorized, msvs

setup.py

@@ -60,16 +60,16 @@ def read_version():
 extras["test"] = (
     [
         extras["all"],
-        "tox==3.13.1",
+        "tox==3.15.1",
         "flake8",
-        "pytest==4.4.1",
+        "pytest==4.6.10",
         "pytest-cov",
         "pytest-rerunfailures",
         "pytest-xdist",
         "mock",
         "contextlib2",
         "awslogs",
-        "black==19.3b0 ; python_version >= '3.6'",
+        "black==19.10b0 ; python_version >= '3.6'",
         "stopit==1.1.2",
         "apache-airflow==1.10.5",
         "fabric>=2.0",

src/sagemaker/amazon/common.py

@@ -261,11 +261,11 @@ def read_recordio(f):
     """
     while True:
         try:
-            read_kmagic, = struct.unpack("I", f.read(4))
+            (read_kmagic,) = struct.unpack("I", f.read(4))
         except struct.error:
             return
         assert read_kmagic == _kmagic
-        len_record, = struct.unpack("I", f.read(4))
+        (len_record,) = struct.unpack("I", f.read(4))
         pad = (((len_record + 3) >> 2) << 2) - len_record
         yield f.read(len_record)
         if pad:

src/sagemaker/cli/common.py

@@ -41,7 +41,7 @@ def __init__(self, args):
         self.script = args.script
         self.instance_type = args.instance_type
         self.instance_count = args.instance_count
-        self.environment = {k: v for k, v in (kv.split("=") for kv in args.env)}
+        self.environment = dict((kv.split("=") for kv in args.env))
 
         self.session = sagemaker.Session()
 

src/sagemaker/rl/estimator.py

@@ -362,10 +362,10 @@ def _validate_framework_format(cls, framework):
         Args:
             framework:
         """
-        if framework and framework not in RLFramework:
+        if framework and framework not in list(RLFramework):
             raise ValueError(
-                "Invalid type: {}, valid RL frameworks types are: [{}]".format(
-                    framework, [t for t in RLFramework]
+                "Invalid type: {}, valid RL frameworks types are: {}".format(
+                    framework, list(RLFramework)
                 )
             )
 
@@ -375,11 +375,9 @@ def _validate_toolkit_format(cls, toolkit):
         Args:
             toolkit:
         """
-        if toolkit and toolkit not in RLToolkit:
+        if toolkit and toolkit not in list(RLToolkit):
             raise ValueError(
-                "Invalid type: {}, valid RL toolkits types are: [{}]".format(
-                    toolkit, [t for t in RLToolkit]
-                )
+                "Invalid type: {}, valid RL toolkits types are: {}".format(toolkit, list(RLToolkit))
             )
 
     @classmethod

src/sagemaker/tuner.py

@@ -92,10 +92,10 @@ def __init__(self, warm_start_type, parents):
                 warm start the new tuning job.
         """
 
-        if warm_start_type not in WarmStartTypes:
+        if warm_start_type not in list(WarmStartTypes):
             raise ValueError(
-                "Invalid type: {}, valid warm start types are: [{}]".format(
-                    warm_start_type, [t for t in WarmStartTypes]
+                "Invalid type: {}, valid warm start types are: {}".format(
+                    warm_start_type, list(WarmStartTypes)
                 )
             )
 

src/sagemaker/workflow/airflow.py

@@ -315,15 +315,17 @@ def tuning_config(tuner, inputs, job_name=None, include_cls_metadata=False, mini
     }
 
     if tuner.estimator:
-        tune_config[
-            "TrainingJobDefinition"
-        ], s3_operations = _extract_training_config_from_estimator(
+        (
+            tune_config["TrainingJobDefinition"],
+            s3_operations,
+        ) = _extract_training_config_from_estimator(
             tuner, inputs, include_cls_metadata, mini_batch_size
         )
     else:
-        tune_config[
-            "TrainingJobDefinitions"
-        ], s3_operations = _extract_training_config_list_from_estimator_dict(
+        (
+            tune_config["TrainingJobDefinitions"],
+            s3_operations,
+        ) = _extract_training_config_list_from_estimator_dict(
             tuner, inputs, include_cls_metadata, mini_batch_size
         )
 

tests/integ/test_monitoring_files.py

@@ -319,7 +319,7 @@ def test_constraint_violations_object_creation_from_file_path_with_customization
 
 
 def test_constraint_violations_object_creation_from_file_path_without_customizations(
-    sagemaker_session
+    sagemaker_session,
 ):
     constraint_violations = ConstraintViolations.from_file_path(
         constraint_violations_file_path=os.path.join(
@@ -354,7 +354,7 @@ def test_constraint_violations_object_creation_from_string_with_customizations(
 
 
 def test_constraint_violations_object_creation_from_string_without_customizations(
-    sagemaker_session
+    sagemaker_session,
 ):
     with open(os.path.join(tests.integ.DATA_DIR, "monitor/constraint_violations.json"), "r") as f:
         file_body = f.read()
@@ -404,7 +404,7 @@ def test_constraint_violations_object_creation_from_s3_uri_with_customizations(
 
 
 def test_constraint_violations_object_creation_from_s3_uri_without_customizations(
-    sagemaker_session
+    sagemaker_session,
 ):
     with open(os.path.join(tests.integ.DATA_DIR, "monitor/constraint_violations.json"), "r") as f:
         file_body = f.read()

tests/integ/test_multi_variant_endpoint.py

@@ -53,8 +53,8 @@
     tests.integ.DATA_DIR, "sparkml_model", "mleap_model.tar.gz"
 )
 SPARK_ML_DEFAULT_VARIANT_NAME = (
-    "AllTraffic"
-)  # default defined in src/sagemaker/session.py def production_variant
+    "AllTraffic"  # default defined in src/sagemaker/session.py def production_variant
+)
 SPARK_ML_WRONG_VARIANT_NAME = "WRONG_VARIANT"
 SPARK_ML_TEST_DATA = "1.0,C,38.0,71.5,1.0,female"
 SPARK_ML_MODEL_SCHEMA = json.dumps(

tests/integ/test_tfs.py

@@ -163,7 +163,7 @@ def test_predict_with_entry_point(tfs_predictor_with_model_and_entry_point_same_
 
 @pytest.mark.local_mode
 def test_predict_with_model_and_entry_point_and_dependencies_separated(
-    tfs_predictor_with_model_and_entry_point_and_dependencies
+    tfs_predictor_with_model_and_entry_point_and_dependencies,
 ):
     input_data = {"instances": [1.0, 2.0, 5.0]}
     expected_result = {"predictions": [4.0, 4.5, 6.0]}

tests/unit/test_image.py

@@ -799,7 +799,7 @@ def test__aws_credentials_with_long_lived_credentials():
 
 @patch("sagemaker.local.image._aws_credentials_available_in_metadata_service")
 def test__aws_credentials_with_short_lived_credentials_and_ec2_metadata_service_having_credentials(
-    mock
+    mock,
 ):
     credentials = Credentials(
         access_key=_random_string(), secret_key=_random_string(), token=_random_string()
@@ -814,7 +814,7 @@ def test__aws_credentials_with_short_lived_credentials_and_ec2_metadata_service_
 
 @patch("sagemaker.local.image._aws_credentials_available_in_metadata_service")
 def test__aws_credentials_with_short_lived_credentials_and_ec2_metadata_service_having_no_credentials(
-    mock
+    mock,
 ):
     credentials = Credentials(
         access_key=_random_string(), secret_key=_random_string(), token=_random_string()

tests/unit/test_tuner.py

@@ -517,7 +517,7 @@ def test_attach_tuning_job_with_estimator_from_hyperparameters(sagemaker_session
 
 
 def test_attach_tuning_job_with_estimator_from_hyperparameters_with_early_stopping(
-    sagemaker_session
+    sagemaker_session,
 ):
     job_details = copy.deepcopy(TUNING_JOB_DETAILS)
     job_details["HyperParameterTuningJobConfig"]["TrainingJobEarlyStoppingType"] = "Auto"

tox.ini

@@ -4,7 +4,7 @@
 # and then run "tox" from this directory.
 
 [tox]
-envlist = black-format,flake8,pylint,twine,sphinx,doc8,py27,py36,py37
+envlist = black-format,flake8,pylint,twine,sphinx,doc8,py27,py36,py37,py38
 
 skip_missing_interpreters = False
 
@@ -80,7 +80,7 @@ basepython = python3
 skipdist = true
 skip_install = true
 deps =
-    pylint==2.3.1
+    pylint==2.5.2
 commands =
     python -m pylint --rcfile=.pylintrc -j 0 src/sagemaker
 
@@ -129,13 +129,13 @@ commands = doc8
 [testenv:black-format]
 # Used during development (before committing) to format .py files.
 basepython = python3
-deps = black==19.3b0
+deps = black==19.10b0
 commands =
     black -l 100 ./
 
 [testenv:black-check]
 # Used by automated build steps to check that all files are properly formatted.
 basepython = python3
-deps = black==19.3b0
+deps = black==19.10b0
 commands =
     black -l 100 --check ./