Processing Jobs Python SDK support (aws#225)

This change adds support for Amazon SageMaker Processing jobs. New classes include Processor, ScriptModeProcessor, SKLearnProcessor, SparkMLJavaProcessor, SparkMLPythonProcessor, ProcessingJob, FileInput, FileOutput, S3Uploader, and S3Downloader.

Author: ajaykarpur

src/sagemaker/fw_utils.py

@@ -19,10 +19,10 @@
 import re
 import shutil
 import tempfile
-from six.moves.urllib.parse import urlparse
 
 import sagemaker.utils
 from sagemaker.utils import get_ecr_image_uri_prefix, ECR_URI_PATTERN
+from sagemaker import s3
 
 _TAR_SOURCE_FILENAME = "source.tar.gz"
 
@@ -447,18 +447,17 @@ def framework_version_from_tag(image_tag):
 
 
 def parse_s3_url(url):
-    """Returns an (s3 bucket, key name/prefix) tuple from a url with an s3
-    scheme
+    """Calls the method with the same name in the s3 module.
+
+    :func:~sagemaker.s3.parse_s3_url
+
     Args:
-        url (str):
-    Returns:
-        tuple: A tuple containing:
-            str: S3 bucket name str: S3 key
+        url: A URL, expected with an s3 scheme.
+
+    Returns: The return value of s3.parse_s3_url, which is a tuple containing:
+        str: S3 bucket name str: S3 key
     """
-    parsed_url = urlparse(url)
-    if parsed_url.scheme != "s3":
-        raise ValueError("Expecting 's3' scheme, got: {} in {}".format(parsed_url.scheme, url))
-    return parsed_url.netloc, parsed_url.path.lstrip("/")
+    return s3.parse_s3_url(url)
 
 
 def model_code_key_prefix(code_location_key_prefix, model_name, image):

src/sagemaker/job.py

@@ -59,6 +59,14 @@ def start_new(self, estimator, inputs):
     def wait(self):
         """Wait for the Amazon SageMaker job to finish."""
 
+    @abstractmethod
+    def describe(self):
+        """Describe the job."""
+
+    @abstractmethod
+    def stop(self):
+        """Stop the job."""
+
     @staticmethod
     def _load_config(inputs, estimator, expand_role=True, validate_uri=True):
         """

src/sagemaker/network.py

@@ -0,0 +1,54 @@
+# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+"""This file contains code related to network configuration, including
+encryption, network isolation, and VPC configurations.
+"""
+from __future__ import absolute_import
+
+
+class NetworkConfig(object):
+    """Accepts network configuration parameters and provides a method to turn these parameters
+    into a dictionary."""
+
+    def __init__(
+        self,
+        enable_network_isolation=False,
+        encrypt_inter_container_traffic=False,
+        security_group_ids=None,
+        subnets=None,
+    ):
+        """Initialize a ``NetworkConfig`` instance. NetworkConfig accepts network configuration
+        parameters and provides a method to turn these parameters into a dictionary.
+
+        Args:
+            enable_network_isolation (bool): Boolean that determines whether to enable
+                network isolation.
+            encrypt_inter_container_traffic (bool): Boolean that determines whether to
+                encrypt inter-container traffic.
+            security_group_ids ([str]): A list of strings representing security group IDs.
+            subnets ([str]): A list of strings representing subnets.
+        """
+        self.enable_network_isolation = enable_network_isolation
+        self.encrypt_inter_container_traffic = encrypt_inter_container_traffic
+        self.security_group_ids = security_group_ids
+        self.subnets = subnets
+
+    def to_request_dict(self):
+        """Generates a request dictionary using the parameters provided to the class."""
+        network_config_request = {
+            "EnableInterContainerTrafficEncryption": self.encrypt_inter_container_traffic,
+            "EnableNetworkIsolation": self.enable_network_isolation,
+            "VpcConfig": {"SecurityGroupIds": self.security_group_ids, "Subnets": self.subnets},
+        }
+
+        return network_config_request