change: allow only one integration test run per time

buildspec.yml

@@ -7,34 +7,48 @@ phases:
 
   build:
     commands:
-      # run linters
-      - tox -e flake8,pylint
-
-      # run package and docbuild checks
-      - tox -e twine
-      - tox -e sphinx
-
-      # run format verification
-      - tox -e black-check
-
-      # run unit tests
-      - AWS_ACCESS_KEY_ID= AWS_SECRET_ACCESS_KEY= AWS_SESSION_TOKEN=
-        AWS_CONTAINER_CREDENTIALS_RELATIVE_URI= AWS_DEFAULT_REGION=
-        tox -e py36,py27 -- tests/unit
-
-      # run notebook test
       - |
-        if has-matching-changes "src/*.py" "setup.py" "setup.cfg" "buildspec.yml"; then
-          echo "running notebook test"
-          ./tests/scripts/run-notebook-test.sh
-         else
-           echo "skipping notebook test"
-         fi
+        python3 ci-scripts/queue_build.py
+        sleep 20m
 
-      # run integration tests
-      - |
-        if has-matching-changes "tests/" "src/*.py" "setup.py" "setup.cfg" "buildspec.yml"; then
-          IGNORE_COVERAGE=- tox -e py36,py27 -- tests/integ -n 24 --boxed --reruns 2
-        else
-          echo "skipping integration tests"
-        fi
+#      # run linters
+#      - tox -e flake8,pylint
+#
+#      # run package and docbuild checks
+#      - tox -e twine
+#      - tox -e sphinx
+#
+#      # run format verification
+#      - tox -e black-check
+#
+#      # run unit tests
+#      - AWS_ACCESS_KEY_ID= AWS_SECRET_ACCESS_KEY= AWS_SESSION_TOKEN=
+#        AWS_CONTAINER_CREDENTIALS_RELATIVE_URI= AWS_DEFAULT_REGION=
+#        tox -e py36,py27 -- tests/unit
+#
+#      # run notebook test
+#      - |
+#        if has-matching-changes "src/*.py" "setup.py" "setup.cfg" "buildspec.yml"; then
+#          echo "running notebook test"
+#          ./tests/scripts/run-notebook-test.sh
+#         else
+#           echo "skipping notebook test"
+#         fi
+#
+#      # wait its turn to run integration tests
+#      - ACCOUNT=$(aws sts get-caller-identity --output text | awk '{print $1}')
+#      - S3_BUCKET_DIR=s3://sagemaker-us-west-2-${ACCOUNT}/ci-lock
+#
+#      # run integration tests
+#      - |
+#        if has-matching-changes "tests/" "src/*.py" "setup.py" "setup.cfg" "buildspec.yml"; then
+#          python ci/queue_build.py
+#          IGNORE_COVERAGE=- tox -e py36,py27 -- tests/integ -n 24 --boxed --reruns 2
+#        else
+#          echo "skipping integration tests"
+#        fi
+    finally:
+      - FILENAME=$(ls ci-lock/)
+      - ACCOUNT=$(aws sts get-caller-identity --output text | awk '{print $1}')
+      - S3_BUCKET_DIR=s3://sagemaker-us-west-2-${ACCOUNT}/ci-lock/
+      - aws s3 rm ${S3_BUCKET_DIR}${FILENAME}

ci-scripts/queue_build.py

@@ -0,0 +1,106 @@
+# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+import os
+import time
+import boto3
+import botocore.exceptions
+
+account = boto3.client("sts").get_caller_identity()["Account"]
+bucket_name = 'sagemaker-us-west-2-%s' % account
+
+
+def queue_build():
+    build_id = os.environ.get('CODEBUILD_BUILD_ID', 'CODEBUILD-BUILD-ID')
+    source_version = os.environ.get('CODEBUILD_SOURCE_VERSION', 'CODEBUILD-SOURCE-VERSION').replace('/', '-')
+    ticket_number = int(time.time())
+    filename = '%s_%s_%s' % (ticket_number, build_id, source_version)
+
+    print('Created queue ticket %s' % ticket_number)
+
+    _write_ticket(filename)
+    files = _list_tickets()
+    _cleanup_tickets_older_than_8_days(files)
+    _wait_for_other_builds(files, ticket_number)
+
+
+def _build_info_from_file(file):
+    filename = file.key.split('/')[1]
+    ticket_number, build_id, source_version = filename.split('_')
+    return int(ticket_number), build_id, source_version
+
+
+def _wait_for_other_builds(files, ticket_number):
+    newfiles = list(filter(lambda file: not _file_older_than(file), files))
+    sorted_files = list(sorted(newfiles, key=lambda y: y.key))
+
+    print('build queue status:')
+    print()
+
+    for order, file in enumerate(sorted_files):
+        file_ticket_number, build_id, source_version = _build_info_from_file(file)
+        print('%s -> %s %s, ticket number: %s' % (order, build_id, source_version, file_ticket_number))
+
+    for file in sorted_files:
+        file_ticket_number, build_id, source_version = _build_info_from_file(file)
+
+        if file_ticket_number == ticket_number:
+
+            break
+        else:
+            while True:
+                try:
+                    print('waiting on build %s %s %s' % (build_id, source_version, file_ticket_number))
+                    file.wait_until_not_exists()
+                    break
+                except botocore.exceptions.WaiterError:
+                    # keep waiting
+                    pass
+
+
+def _cleanup_tickets_older_than_8_days(files):
+    oldfiles = list(filter(_file_older_than, files))
+    for file in oldfiles:
+        print('object %s older than 8 hours. Deleting' % file.key)
+        file.delete()
+    return files
+
+
+def _list_tickets():
+    s3 = boto3.resource('s3')
+    bucket = s3.Bucket(bucket_name)
+    objects = [file for file in bucket.objects.filter(Prefix='ci-lock/')]
+    files = list(filter(lambda x: x != 'ci-lock/', objects))
+    return files
+
+
+def _file_older_than(file):
+    timelimit = 60 * 60 * 8
+
+    file_ticket_number, build_id, source_version = _build_info_from_file(file)
+
+    return int(time.time()) - file_ticket_number > timelimit
+
+
+def _write_ticket(ticket_number):
+
+    if not os.path.exists('ci-lock'):
+        os.mkdir('ci-lock')
+
+    filename = 'ci-lock/' + ticket_number
+    with open(filename, 'w') as file:
+        file.write(ticket_number)
+    boto3.Session().resource("s3").Object(bucket_name, filename).upload_file(filename)
+
+
+if __name__ == '__main__':
+    queue_build()