Merge branch 'master' into iss96242

Author: icywang86rui

CHANGELOG.md

@@ -1,5 +1,16 @@
 # Changelog
 
+## v2.23.2 (2021-01-06)
+
+### Bug Fixes and Other Changes
+
+ * remove shell=True in subprocess.check_output
+ * use SecurityConfig dict key
+
+### Documentation Changes
+
+ * remove D212 from ignore to comply with PEP257 standards
+
 ## v2.23.1 (2020-12-29)
 
 ### Bug Fixes and Other Changes

VERSION

@@ -1 +1 @@
-2.23.2.dev0
+2.23.3.dev0

doc/v2.rst

@@ -318,9 +318,9 @@ The follow serializer/deserializer classes have been renamed and/or moved:
 +--------------------------------------------------------+-------------------------------------------------------+
 | ``sagemaker.predictor._NPYSerializer``                 | ``sagemaker.serializers.NumpySerializer``             |
 +--------------------------------------------------------+-------------------------------------------------------+
-| ``sagemaker.amazon.common.numpy_to_record_serializer`` | ``sagemaker.amazon.serializers.RecordSerializer``     |
+| ``sagemaker.amazon.common.numpy_to_record_serializer`` | ``sagemaker.amazon.common.RecordSerializer``          |
 +--------------------------------------------------------+-------------------------------------------------------+
-| ``sagemaker.amazon.common.record_deserializer``        | ``sagemaker.amazon.deserializers.RecordDeserializer`` |
+| ``sagemaker.amazon.common.record_deserializer``        | ``sagemaker.amazon.common.RecordDeserializer``        |
 +--------------------------------------------------------+-------------------------------------------------------+
 | ``sagemaker.predictor._JsonDeserializer``              | ``sagemaker.deserializers.JSONDeserializer``          |
 +--------------------------------------------------------+-------------------------------------------------------+

src/sagemaker/feature_store/feature_group.py

@@ -152,7 +152,7 @@ class IngestionManagerPandas:
         feature_group_name (str): name of the Feature Group.
         sagemaker_session (Session): instance of the Session class to perform boto calls.
         data_frame (DataFrame): pandas DataFrame to be ingested to the given feature group.
-        max_works (int): number of threads to create.
+        max_workers (int): number of threads to create.
     """
 
     feature_group_name: str = attr.ib()
@@ -508,6 +508,9 @@ def as_hive_ddl(self, database: str = "sagemaker_featurestore", table_name: str
                 f"  {definition.feature_name} "
                 f"{self._FEATURE_TYPE_TO_DDL_DATA_TYPE_MAP.get(definition.feature_type.value)}\n"
             )
+        ddl += "  write_time TIMESTAMP\n"
+        ddl += "  event_time TIMESTAMP\n"
+        ddl += "  is_deleted BOOLEAN\n"
         ddl += ")\n"
         ddl += (
             "ROW FORMAT SERDE 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe'\n"

src/sagemaker/local/image.py

@@ -1067,7 +1067,7 @@ def _ecr_login_if_needed(boto_session, image):
     ecr_url = auth["authorizationData"][0]["proxyEndpoint"]
 
     cmd = "docker login -u AWS -p %s %s" % (token, ecr_url)
-    subprocess.check_output(cmd, shell=True)
+    subprocess.check_output(cmd.split())
 
     return True
 
@@ -1081,5 +1081,5 @@ def _pull_image(image):
     pull_image_command = ("docker pull %s" % image).strip()
     logger.info("docker command: %s", pull_image_command)
 
-    subprocess.check_output(pull_image_command, shell=True)
+    subprocess.check_output(pull_image_command.split())
     logger.info("image pulled: %s", image)

tests/integ/test_feature_store.py

@@ -131,7 +131,11 @@ def create_table_ddl():
         "CREATE EXTERNAL TABLE IF NOT EXISTS sagemaker_featurestore.{feature_group_name} (\n"
         "  feature1 FLOAT\n"
         "  feature2 INT\n"
-        "  feature3 STRING\n)\n"
+        "  feature3 STRING\n"
+        "  write_time TIMESTAMP\n"
+        "  event_time TIMESTAMP\n"
+        "  is_deleted BOOLEAN\n"
+        ")\n"
         "ROW FORMAT SERDE 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe'\n"
         "  STORED AS\n"
         "  INPUTFORMAT 'parquet.hive.DeprecatedParquetInputFormat'\n"

tests/integ/test_workflow.py

@@ -506,15 +506,17 @@ def test_training_job_with_debugger(
         except WaiterError:
             pass
         execution_steps = execution.list_steps()
-        training_job_arn = execution_steps[0]["Metadata"]["TrainingJob"]["Arn"]
-        job_description = sagemaker_session.sagemaker_client.describe_training_job(
-            TrainingJobName=training_job_arn.split("/")[1]
-        )
 
         assert len(execution_steps) == 1
+        assert execution_steps[0].get("FailureReason", "") == ""
         assert execution_steps[0]["StepName"] == "pytorch-train"
         assert execution_steps[0]["StepStatus"] == "Succeeded"
 
+        training_job_arn = execution_steps[0]["Metadata"]["TrainingJob"]["Arn"]
+        job_description = sagemaker_session.sagemaker_client.describe_training_job(
+            TrainingJobName=training_job_arn.split("/")[1]
+        )
+
         for index, rule in enumerate(rules):
             config = job_description["DebugRuleConfigurations"][index]
             assert config["RuleConfigurationName"] == rule.name

tests/unit/sagemaker/feature_store/test_feature_store.py

@@ -57,7 +57,11 @@ def create_table_ddl():
         "CREATE EXTERNAL TABLE IF NOT EXISTS {database}.{table_name} (\n"
         "  feature1 FLOAT\n"
         "  feature2 INT\n"
-        "  feature3 STRING\n)\n"
+        "  feature3 STRING\n"
+        "  write_time TIMESTAMP\n"
+        "  event_time TIMESTAMP\n"
+        "  is_deleted BOOLEAN\n"
+        ")\n"
         "ROW FORMAT SERDE 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe'\n"
         "  STORED AS\n"
         "  INPUTFORMAT 'parquet.hive.DeprecatedParquetInputFormat'\n"

tests/unit/test_image.py

@@ -765,7 +765,7 @@ def test_ecr_login_needed(check_output):
         "docker login -u AWS -p %s https://520713654638.dkr.ecr.us-east-1.amazonaws.com" % token
     )
 
-    check_output.assert_called_with(expected_command, shell=True)
+    check_output.assert_called_with(expected_command.split())
     session_mock.client("ecr").get_authorization_token.assert_called_with(
         registryIds=["520713654638"]
     )
@@ -781,7 +781,7 @@ def test_pull_image(check_output):
 
     expected_command = "docker pull %s" % image
 
-    check_output.assert_called_once_with(expected_command, shell=True)
+    check_output.assert_called_once_with(expected_command.split())
 
 
 def test__aws_credentials_with_long_lived_credentials():