Merge branch 'dev' into patch-1

Author: shreyapandit

CHANGELOG.md

@@ -1,5 +1,51 @@
 # Changelog
 
+## v2.72.0 (2021-12-13)
+
+### Features
+
+ * allow conditional parellel builds
+
+### Bug Fixes and Other Changes
+
+ * local mode - support relative file structure
+ * fix endpoint bug
+
+## v2.71.0 (2021-12-06)
+
+### Features
+
+ * Add support for TF 2.6
+ * Adding PT 17/18 Repo
+ * Add profile_name support for Feature Store ingestion
+
+### Bug Fixes and Other Changes
+
+ * Fix non-existent variable name
+ * Add TF 2.6.2 on training
+ * Recreate static lineage test data
+
+## v2.70.0 (2021-12-02)
+
+### Features
+
+ * update boto3 minor version >= 1.20.18
+ * Add support for SageMaker lineage queries
+ * add CV shap explainability for SageMaker Clarify
+ * add NLP support for SageMaker Clarify
+ * Add support for ModelMonitor/Clarify integration in model building pipelines
+ * adding support for transformers 4.11 for SM Training Compiler
+ * SM Training Compiler with an UI to enable/disable compilation for HuggingFace DLCs to speedup training
+
+### Bug Fixes and Other Changes
+
+ * pin coveragepy
+ * Add support for PyTorch 1.9.1
+ * Update s3 path of scheduling analysis config on ClarifyCheckStep
+ * documentation/logging to indicate correct place for DEBUG artifacts from SM trcomp
+ * validate requested transformers version and use the best available version
+ * Install custom pkgs
+
 ## v2.69.0 (2021-11-12)
 
 ### Features

VERSION

@@ -1 +1 @@
-2.69.1.dev0
+2.72.1.dev0

ci-scripts/queue_build.py

@@ -23,100 +23,138 @@
 ).get_caller_identity()["Account"]
 bucket_name = "sagemaker-us-west-2-%s" % account
 
+MAX_IN_PROGRESS_BUILDS = 3
+INTERVAL_BETWEEN_CONCURRENT_RUNS = 15  # minutes
+CLEAN_UP_TICKETS_OLDER_THAN = 8  # hours
+
 
 def queue_build():
-    build_id = re.sub("[_/]", "-", os.environ.get("CODEBUILD_BUILD_ID", "CODEBUILD-BUILD-ID"))
-    source_version = re.sub(
-        "[_/]",
-        "-",
-        os.environ.get("CODEBUILD_SOURCE_VERSION", "CODEBUILD-SOURCE-VERSION"),
-    )
     ticket_number = int(1000 * time.time())
-    filename = "%s_%s_%s" % (ticket_number, build_id, source_version)
-
-    print("Created queue ticket %s" % ticket_number)
-
-    _write_ticket(filename)
     files = _list_tickets()
-    _cleanup_tickets_older_than_8_hours(files)
-    _wait_for_other_builds(files, ticket_number)
+    _cleanup_tickets_older_than(files)
+    _wait_for_other_builds(ticket_number)
 
 
 def _build_info_from_file(file):
-    filename = file.key.split("/")[1]
+    filename = file.key.split("/")[2]
     ticket_number, build_id, source_version = filename.split("_")
     return int(ticket_number), build_id, source_version
 
 
-def _wait_for_other_builds(files, ticket_number):
-    newfiles = list(filter(lambda file: not _file_older_than(file), files))
-    sorted_files = list(sorted(newfiles, key=lambda y: y.key))
+def _wait_for_other_builds(ticket_number):
+    sorted_files = _list_tickets()
 
     print("build queue status:")
     print()
 
     for order, file in enumerate(sorted_files):
         file_ticket_number, build_id, source_version = _build_info_from_file(file)
         print(
-            "%s -> %s %s, ticket number: %s" % (order, build_id, source_version, file_ticket_number)
+            "%s -> %s %s, ticket number: %s status: %s"
+            % (order, build_id, source_version, file_ticket_number, file.key.split("/")[1])
         )
+    print()
+    build_id = re.sub("[_/]", "-", os.environ.get("CODEBUILD_BUILD_ID", "CODEBUILD-BUILD-ID"))
+    source_version = re.sub(
+        "[_/]",
+        "-",
+        os.environ.get("CODEBUILD_SOURCE_VERSION", "CODEBUILD-SOURCE-VERSION"),
+    )
+    filename = "%s_%s_%s" % (ticket_number, build_id, source_version)
+    s3_file_obj = _write_ticket(filename, status="waiting")
+    print("Build %s waiting to be scheduled" % filename)
+
+    while True:
+        _cleanup_tickets_with_terminal_states()
+        waiting_tickets = _list_tickets("waiting")
+        if waiting_tickets:
+            first_waiting_ticket_number, _, _ = _build_info_from_file(_list_tickets("waiting")[0])
+        else:
+            first_waiting_ticket_number = ticket_number
+
+        if (
+            len(_list_tickets(status="in-progress")) < 3
+            and last_in_progress_elapsed_time_check()
+            and first_waiting_ticket_number == ticket_number
+        ):
+            # put the build in progress
+            print("Scheduling build %s for running.." % filename)
+            s3_file_obj.delete()
+            _write_ticket(filename, status="in-progress")
+            break
+        else:
+            # wait
+            time.sleep(30)
 
-    for file in sorted_files:
-        file_ticket_number, build_id, source_version = _build_info_from_file(file)
 
-        if file_ticket_number == ticket_number:
+def last_in_progress_elapsed_time_check():
+    in_progress_tickets = _list_tickets("in-progress")
+    if not in_progress_tickets:
+        return True
+    last_in_progress_ticket, _, _ = _build_info_from_file(_list_tickets("in-progress")[-1])
+    _elapsed_time = int(1000 * time.time()) - last_in_progress_ticket
+    last_in_progress_elapsed_time = int(_elapsed_time / (1000 * 60))  # in minutes
+    return last_in_progress_elapsed_time > INTERVAL_BETWEEN_CONCURRENT_RUNS
 
-            break
-        else:
-            while True:
-                client = boto3.client("codebuild")
-                response = client.batch_get_builds(ids=[build_id])
-                build_status = response["builds"][0]["buildStatus"]
-
-                if build_status == "IN_PROGRESS":
-                    print(
-                        "waiting on build %s %s %s" % (build_id, source_version, file_ticket_number)
-                    )
-                    time.sleep(30)
-                else:
-                    print("build %s finished, deleting lock" % build_id)
-                    file.delete()
-                    break
-
-
-def _cleanup_tickets_older_than_8_hours(files):
+
+def _cleanup_tickets_with_terminal_states():
+    files = _list_tickets()
+    build_ids = []
+    for file in files:
+        _, build_id, _ = _build_info_from_file(file)
+        build_ids.append(build_id)
+
+    client = boto3.client("codebuild")
+    response = client.batch_get_builds(ids=build_ids)
+
+    for file, build_details in zip(files, response["builds"]):
+        _, _build_id_from_file, _ = _build_info_from_file(file)
+        build_status = build_details["buildStatus"]
+
+        if build_status != "IN_PROGRESS" and _build_id_from_file == build_details["id"]:
+            print(
+                "Build %s in terminal state: %s, deleting lock"
+                % (_build_id_from_file, build_status)
+            )
+            file.delete()
+
+
+def _cleanup_tickets_older_than(files):
     oldfiles = list(filter(_file_older_than, files))
     for file in oldfiles:
         print("object %s older than 8 hours. Deleting" % file.key)
         file.delete()
     return files
 
 
-def _list_tickets():
+def _list_tickets(status=None):
     s3 = boto3.resource("s3")
     bucket = s3.Bucket(bucket_name)
-    objects = [file for file in bucket.objects.filter(Prefix="ci-lock/")]
-    files = list(filter(lambda x: x != "ci-lock/", objects))
-    return files
+    prefix = "ci-integ-queue/{}/".format(status) if status else "ci-integ-queue/"
+    objects = [file for file in bucket.objects.filter(Prefix=prefix)]
+    files = list(filter(lambda x: x != prefix, objects))
+    sorted_files = list(sorted(files, key=lambda y: y.key))
+    return sorted_files
 
 
 def _file_older_than(file):
-    timelimit = 1000 * 60 * 60 * 8
-
+    timelimit = 1000 * 60 * 60 * CLEAN_UP_TICKETS_OLDER_THAN
     file_ticket_number, build_id, source_version = _build_info_from_file(file)
+    return int(1000 * time.time()) - file_ticket_number > timelimit
 
-    return int(time.time()) - file_ticket_number > timelimit
-
-
-def _write_ticket(ticket_number):
 
-    if not os.path.exists("ci-lock"):
-        os.mkdir("ci-lock")
+def _write_ticket(filename, status="waiting"):
+    file_path = "ci-integ-queue/{}".format(status)
+    if not os.path.exists(file_path):
+        os.makedirs(file_path)
 
-    filename = "ci-lock/" + ticket_number
-    with open(filename, "w") as file:
-        file.write(ticket_number)
-    boto3.Session().resource("s3").Object(bucket_name, filename).upload_file(filename)
+    file_full_path = file_path + "/" + filename
+    with open(file_full_path, "w") as file:
+        file.write(filename)
+    s3_file_obj = boto3.Session().resource("s3").Object(bucket_name, file_full_path)
+    s3_file_obj.upload_file(file_full_path)
+    print("Build %s is now in state %s" % (filename, status))
+    return s3_file_obj
 
 
 if __name__ == "__main__":

doc/api/inference/model_monitor.rst

@@ -31,3 +31,13 @@ Model Monitor
     :members:
     :undoc-members:
     :show-inheritance:
+
+.. automodule:: sagemaker.model_metrics
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+.. automodule:: sagemaker.drift_check_baselines
+    :members:
+    :undoc-members:
+    :show-inheritance:

doc/frameworks/huggingface/sagemaker.huggingface.rst

@@ -4,7 +4,15 @@ Hugging Face
 Hugging Face Estimator
 ----------------------
 
-.. autoclass:: sagemaker.huggingface.estimator.HuggingFace
+.. autoclass:: sagemaker.huggingface.HuggingFace
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+Hugging Face Training Compiler Configuration
+--------------------------------------------
+
+.. autoclass:: sagemaker.huggingface.TrainingCompilerConfig
     :members:
     :undoc-members:
     :show-inheritance:
@@ -17,8 +25,8 @@ Hugging Face Model
     :undoc-members:
     :show-inheritance:
 
-HuggingFace Predictor
----------------------
+Hugging Face Predictor
+----------------------
 
 .. autoclass:: sagemaker.huggingface.model.HuggingFacePredictor
     :members:

doc/workflows/pipelines/sagemaker.workflow.pipelines.rst

@@ -32,6 +32,11 @@ Conditions
 
 .. autoclass:: sagemaker.workflow.conditions.ConditionOr
 
+CheckJobConfig
+--------------
+
+.. autoclass:: sagemaker.workflow.check_job_config.CheckJobConfig
+
 Entities
 --------
 
@@ -128,3 +133,13 @@ Steps
 .. autoclass:: sagemaker.workflow.steps.CacheConfig
 
 .. autoclass:: sagemaker.workflow.lambda_step.LambdaStep
+
+.. autoclass:: sagemaker.workflow.steps.CompilationStep
+
+.. autoclass:: sagemaker.workflow.quality_check_step.QualityCheckConfig
+
+.. autoclass:: sagemaker.workflow.quality_check_step.QualityCheckStep
+
+.. autoclass:: sagemaker.workflow.clarify_check_step.ClarifyCheckConfig
+
+.. autoclass:: sagemaker.workflow.clarify_check_step.ClarifyCheckStep

setup.py

@@ -34,7 +34,7 @@ def read_version():
 # Declare minimal set for installation
 required_packages = [
     "attrs",
-    "boto3>=1.16.32",
+    "boto3>=1.20.18",
     "google-pasta",
     "numpy>=1.9.0",
     "protobuf>=3.1",
@@ -69,6 +69,7 @@ def read_version():
         "pytest-rerunfailures",
         "pytest-timeout",
         "pytest-xdist",
+        "coverage<6.2",
         "mock",
         "contextlib2",
         "awslogs",

src/sagemaker/__init__.py

@@ -49,7 +49,7 @@
 from sagemaker.local.local_session import LocalSession  # noqa: F401
 
 from sagemaker.model import Model, ModelPackage  # noqa: F401
-from sagemaker.model_metrics import ModelMetrics, MetricsSource  # noqa: F401
+from sagemaker.model_metrics import ModelMetrics, MetricsSource, FileSource  # noqa: F401
 from sagemaker.pipeline import PipelineModel  # noqa: F401
 from sagemaker.predictor import Predictor  # noqa: F401
 from sagemaker.processing import Processor, ScriptProcessor  # noqa: F401