diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e25a373c..65cc6005 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,16 +13,34 @@ env: PACKAGE_NAME: aws-iot-device-sdk-python-v2 LINUX_BASE_IMAGE: ubuntu-16-x64 RUN: ${{ github.run_id }}-${{ github.run_number }} - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_DATEST_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_DATEST_SECRET_ACCESS_KEY }} AWS_DEFAULT_REGION: us-east-1 + DA_TOPIC: test/da + DA_SHADOW_PROPERTY: datest + DA_SHADOW_VALUE_SET: ON + DA_SHADOW_VALUE_DEFAULT: OFF + CI_UTILS_FOLDER: "./aws-iot-device-sdk-python-v2/utils" + CI_SAMPLES_FOLDER: "./aws-iot-device-sdk-python-v2/samples" + CI_IOT_CONTAINERS: ${{ secrets.AWS_CI_IOT_CONTAINERS }} + CI_PUBSUB_ROLE: ${{ secrets.AWS_CI_PUBSUB_ROLE }} + CI_CUSTOM_AUTHORIZER_ROLE: ${{ secrets.AWS_CI_CUSTOM_AUTHORIZER_ROLE }} + CI_SHADOW_ROLE: ${{ secrets.AWS_CI_SHADOW_ROLE }} + CI_JOBS_ROLE: ${{ secrets.AWS_CI_JOBS_ROLE }} + CI_FLEET_PROVISIONING_ROLE: ${{ secrets.AWS_CI_FLEET_PROVISIONING_ROLE }} + CI_DEVICE_ADVISOR: ${{ secrets.AWS_CI_DEVICE_ADVISOR_ROLE }} jobs: al2: runs-on: ubuntu-latest + permissions: + id-token: write # This is required for requesting the JWT steps: - # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages + - name: configure AWS credentials (containers) + uses: aws-actions/configure-aws-credentials@v1 + with: + role-to-assume: ${{ env.CI_IOT_CONTAINERS }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} + # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages - name: Build ${{ env.PACKAGE_NAME }} run: | aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh @@ -30,20 +48,171 @@ jobs: windows: runs-on: windows-latest + permissions: + id-token: write # This is required for requesting the JWT steps: - name: Build ${{ env.PACKAGE_NAME }} run: | python -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder.pyz')" python builder.pyz build -p ${{ env.PACKAGE_NAME }} + - name: Running samples in CI setup + run: | + python -m pip install boto3 + - name: configure AWS credentials (PubSub) + uses: aws-actions/configure-aws-credentials@v1 + with: + role-to-assume: ${{ env.CI_PUBSUB_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} + - name: run PubSub sample + run: | + python ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --language Python --sample_file "${{ env.CI_SAMPLES_FOLDER }}/pubsub.py" --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/PubSub/cert' --sample_secret_private_key 'ci/PubSub/key' + - name: run Windows Certificate Connect sample + run: | + python ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --language Python --sample_file "${{ env.CI_SAMPLES_FOLDER }}/windows_cert_connect.py" --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/PubSub/cert' --sample_secret_private_key 'ci/PubSub/key' --sample_run_certutil true + - name: configure AWS credentials (Device Advisor) + uses: aws-actions/configure-aws-credentials@v1 + with: + role-to-assume: ${{ env.CI_DEVICE_ADVISOR }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} + - name: run DeviceAdvisor + run: | + cd ./aws-iot-device-sdk-python-v2 + python ./deviceadvisor/script/DATestRun.py osx: runs-on: macos-latest + permissions: + id-token: write # This is required for requesting the JWT + steps: + - name: Build ${{ env.PACKAGE_NAME }} + run: | + python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')" + chmod a+x builder + ./builder build -p ${{ env.PACKAGE_NAME }} + - name: Running samples in CI setup + run: | + python3 -m pip install boto3 + - name: configure AWS credentials (PubSub) + uses: aws-actions/configure-aws-credentials@v1 + with: + role-to-assume: ${{ env.CI_PUBSUB_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} + - name: run PubSub sample + run: | + python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --language Python --sample_file "${{ env.CI_SAMPLES_FOLDER }}/pubsub.py" --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/PubSub/cert' --sample_secret_private_key 'ci/PubSub/key' + - name: configure AWS credentials (Device Advisor) + uses: aws-actions/configure-aws-credentials@v1 + with: + role-to-assume: ${{ env.CI_DEVICE_ADVISOR }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} + - name: run DeviceAdvisor + run: | + cd ./aws-iot-device-sdk-python-v2 + python3 ./deviceadvisor/script/DATestRun.py + + linux: + runs-on: ubuntu-20.04 # latest + permissions: + id-token: write # This is required for requesting the JWT steps: - name: Build ${{ env.PACKAGE_NAME }} run: | python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')" chmod a+x builder ./builder build -p ${{ env.PACKAGE_NAME }} + - name: Running samples in CI setup + run: | + python3 -m pip install boto3 + - name: configure AWS credentials (PubSub) + uses: aws-actions/configure-aws-credentials@v1 + with: + role-to-assume: ${{ env.CI_PUBSUB_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} + - name: run PubSub sample + run: | + python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --language Python --sample_file "${{ env.CI_SAMPLES_FOLDER }}/pubsub.py" --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/PubSub/cert' --sample_secret_private_key 'ci/PubSub/key' + - name: configure AWS credentials (Device Advisor) + uses: aws-actions/configure-aws-credentials@v1 + with: + role-to-assume: ${{ env.CI_DEVICE_ADVISOR }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} + - name: run DeviceAdvisor + run: | + cd ./aws-iot-device-sdk-python-v2 + python3 ./deviceadvisor/script/DATestRun.py + + # Runs the samples and ensures that everything is working + linux-smoke-tests: + runs-on: ubuntu-latest + permissions: + id-token: write # This is required for requesting the JWT + steps: + - name: Build ${{ env.PACKAGE_NAME }} + run: | + python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')" + chmod a+x builder + ./builder build -p ${{ env.PACKAGE_NAME }} + - name: Running samples in CI setup + run: | + python3 -m pip install boto3 + sudo apt-get update -y + sudo apt-get install softhsm -y + softhsm2-util --version + - name: configure AWS credentials (Connect and PubSub) + uses: aws-actions/configure-aws-credentials@v1 + with: + role-to-assume: ${{ env.CI_PUBSUB_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} + - name: run Basic Connect sample + run: | + python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --language Python --sample_file "${{ env.CI_SAMPLES_FOLDER }}/basic_connect.py" --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/PubSub/cert' --sample_secret_private_key 'ci/PubSub/key' + - name: run Websocket Connect sample + run: | + python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --language Python --sample_file "${{ env.CI_SAMPLES_FOLDER }}/websocket_connect.py" --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_arguments '--signing_region us-east-1' + - name: run PubSub sample + run: | + python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --language Python --sample_file "${{ env.CI_SAMPLES_FOLDER }}/pubsub.py" --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/PubSub/cert' --sample_secret_private_key 'ci/PubSub/key' + - name: run PKCS11 Connect sample + run: | + mkdir -p /tmp/tokens + export SOFTHSM2_CONF=/tmp/softhsm2.conf + echo "directories.tokendir = /tmp/tokens" > /tmp/softhsm2.conf + python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --language Python --sample_file "${{ env.CI_SAMPLES_FOLDER }}/pkcs11_connect.py" --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/PubSub/cert' --sample_secret_private_key 'ci/PubSub/keyp8' --sample_run_softhsm 'true' --sample_arguments '--pkcs11_lib /usr/lib/softhsm/libsofthsm2.so --pin 0000 --token_label my-token --key_label my-key' + - name: configure AWS credentials (Custom Authorizer) + uses: aws-actions/configure-aws-credentials@v1 + with: + role-to-assume: ${{ env.CI_CUSTOM_AUTHORIZER_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} + - name: run CustomAuthorizerConnect sample + run: | + python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --language Python --sample_file "${{ env.CI_SAMPLES_FOLDER }}/custom_authorizer_connect.py" --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_custom_authorizer_name 'ci/CustomAuthorizer/name' --sample_secret_custom_authorizer_password 'ci/CustomAuthorizer/password' + - name: configure AWS credentials (Shadow) + uses: aws-actions/configure-aws-credentials@v1 + with: + role-to-assume: ${{ env.CI_SHADOW_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} + - name: run Shadow sample + run: | + python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --language Python --sample_file "${{ env.CI_SAMPLES_FOLDER }}/shadow.py" --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/Shadow/cert' --sample_secret_private_key 'ci/Shadow/key' --sample_arguments '--thing_name CI_Shadow_Thing' + - name: configure AWS credentials (Jobs) + uses: aws-actions/configure-aws-credentials@v1 + with: + role-to-assume: ${{ env.CI_JOBS_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} + - name: run Jobs sample + run: | + python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --language Python --sample_file "${{ env.CI_SAMPLES_FOLDER }}/jobs.py" --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/Jobs/cert' --sample_secret_private_key 'ci/Jobs/key' --sample_arguments '--thing_name CI_Jobs_Thing' + - name: configure AWS credentials (Fleet provisioning) + uses: aws-actions/configure-aws-credentials@v1 + with: + role-to-assume: ${{ env.CI_FLEET_PROVISIONING_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} + - name: run Fleet Provisioning sample + run: | + echo "Generating UUID for IoT thing" + Sample_UUID=$(python3 -c "import uuid; print (uuid.uuid4())") + python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --language Python --sample_file "${{ env.CI_SAMPLES_FOLDER }}/fleetprovisioning.py" --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/FleetProvisioning/cert' --sample_secret_private_key 'ci/FleetProvisioning/key' --sample_arguments "--template_name CI_FleetProvisioning_Template --template_parameters {\"SerialNumber\":\"${Sample_UUID}\"}" + python3 ${{ env.CI_UTILS_FOLDER }}/delete_iot_thing_ci.py --thing_name "Fleet_Thing_${Sample_UUID}" --region "us-east-1" # check that docs can still build check-docs: diff --git a/README.md b/README.md index d31c9af4..b654be60 100644 --- a/README.md +++ b/README.md @@ -36,10 +36,19 @@ to Python by the `awscrt` package ([PyPI](https://pypi.org/project/awscrt/)) ([G [Step-by-step instructions](./documents/PREREQUISITES.md) ### Install from PyPI + +#### MacOS and Linux: + ``` python3 -m pip install awsiotsdk ``` +#### Windows: + +``` +python -m pip install awsiotsdk +``` + ### Install from source ``` # Create a workspace directory to hold all the SDK files @@ -47,7 +56,7 @@ mkdir sdk-workspace cd sdk-workspace # Clone the repository git clone https://github.com/aws/aws-iot-device-sdk-python-v2.git -# Install using Pip +# Install using Pip (use 'python' instead of 'python3' on Windows) python3 -m pip install ./aws-iot-device-sdk-python-v2 ``` diff --git a/builder.json b/builder.json index 2ce80e82..f1a0a956 100644 --- a/builder.json +++ b/builder.json @@ -89,14 +89,7 @@ "python3 -m pip install ." ], "test_steps": [ - "python3 -m pip install boto3", - "python3 deviceadvisor/script/DATestRun.py" ], "env": { - "DA_TOPIC": "test/da", - "DA_SHADOW_PROPERTY": "datest", - "DA_SHADOW_VALUE_SET": "ON", - "DA_SHADOW_VALUE_DEFAULT": "OFF", - "DA_S3_NAME": "aws-iot-sdk-deviceadvisor-logs" } } diff --git a/codebuild/samples/connect-auth-linux.sh b/codebuild/samples/connect-auth-linux.sh deleted file mode 100755 index 0027ce3a..00000000 --- a/codebuild/samples/connect-auth-linux.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash - -set -e - -env - -pushd $CODEBUILD_SRC_DIR/samples/ - -ENDPOINT=$(aws secretsmanager get-secret-value --secret-id "unit-test/endpoint" --query "SecretString" | cut -f2 -d":" | sed -e 's/[\\\"\}]//g') -AUTH_NAME=$(aws secretsmanager get-secret-value --secret-id "unit-test/authorizer-name" --query "SecretString" | cut -f2 -d":" | sed -e 's/[\\\"\}]//g') -AUTH_PASSWORD=$(aws secretsmanager get-secret-value --secret-id "unit-test/authorizer-password" --query "SecretString" | cut -f2 -d":" | sed -e 's/[\\\"\}]//g') - -echo "Custom authorizer connect test" -python3 custom_authorizer_connect.py --endpoint $ENDPOINT --custom_auth_authorizer_name $AUTH_NAME --custom_auth_password $AUTH_PASSWORD - -popd diff --git a/codebuild/samples/connect-linux.sh b/codebuild/samples/connect-linux.sh deleted file mode 100755 index ab23b0a7..00000000 --- a/codebuild/samples/connect-linux.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash - -set -e - -env - -pushd $CODEBUILD_SRC_DIR/samples/ - -ENDPOINT=$(aws secretsmanager get-secret-value --secret-id "unit-test/endpoint" --query "SecretString" | cut -f2 -d":" | sed -e 's/[\\\"\}]//g') - -echo "Mqtt Direct test" -python3 basic_connect.py --endpoint $ENDPOINT --key /tmp/privatekey.pem --cert /tmp/certificate.pem - -echo "Websocket test" -python3 websocket_connect.py --endpoint $ENDPOINT --signing_region us-east-1 - -popd diff --git a/codebuild/samples/linux-smoke-tests.yml b/codebuild/samples/linux-smoke-tests.yml index 0e4ddc94..7fec9b56 100644 --- a/codebuild/samples/linux-smoke-tests.yml +++ b/codebuild/samples/linux-smoke-tests.yml @@ -9,10 +9,7 @@ phases: commands: - echo Build started on `date` - $CODEBUILD_SRC_DIR/codebuild/samples/setup-linux.sh - - $CODEBUILD_SRC_DIR/codebuild/samples/connect-linux.sh - - $CODEBUILD_SRC_DIR/codebuild/samples/pkcs11-connect-linux.sh - $CODEBUILD_SRC_DIR/codebuild/samples/pubsub-linux.sh - - $CODEBUILD_SRC_DIR/codebuild/samples/connect-auth-linux.sh post_build: commands: - echo Build completed on `date` diff --git a/codebuild/samples/pkcs11-connect-linux.sh b/codebuild/samples/pkcs11-connect-linux.sh deleted file mode 100755 index b1f80c44..00000000 --- a/codebuild/samples/pkcs11-connect-linux.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/bash - -set -e -set -o pipefail - -pushd $CODEBUILD_SRC_DIR/samples/ - -ENDPOINT=$(aws secretsmanager get-secret-value --secret-id "unit-test/endpoint" --query "SecretString" | cut -f2 -d":" | sed -e 's/[\\\"\}]//g') - -# from hereon commands are echoed. don't leak secrets -set -x - -softhsm2-util --version - -# SoftHSM2's default tokendir path might be invalid on this machine -# so set up a conf file that specifies a known good tokendir path -mkdir -p /tmp/tokens -export SOFTHSM2_CONF=/tmp/softhsm2.conf -echo "directories.tokendir = /tmp/tokens" > /tmp/softhsm2.conf - -# create token -softhsm2-util --init-token --free --label my-token --pin 0000 --so-pin 0000 - -# add private key to token (must be in PKCS#8 format) -openssl pkcs8 -topk8 -in /tmp/privatekey.pem -out /tmp/privatekey.p8.pem -nocrypt -softhsm2-util --import /tmp/privatekey.p8.pem --token my-token --label my-key --id BEEFCAFE --pin 0000 - -# run sample -python3 pkcs11_connect.py \ - --endpoint $ENDPOINT \ - --cert /tmp/certificate.pem \ - --pkcs11_lib /usr/lib/softhsm/libsofthsm2.so \ - --pin 0000 \ - --token_label my-token \ - --key_label my-key - -popd diff --git a/deviceadvisor/script/DATestConfig.json b/deviceadvisor/script/DATestConfig.json index 01e8e0af..af5c5798 100644 --- a/deviceadvisor/script/DATestConfig.json +++ b/deviceadvisor/script/DATestConfig.json @@ -2,11 +2,11 @@ "tests" :["MQTT Connect", "MQTT Publish", "MQTT Subscribe", "Shadow Publish", "Shadow Update"], "test_suite_ids" : { - "MQTT Connect" : "ejbdzmo3hf3v", - "MQTT Publish" : "euw7favf6an4", - "MQTT Subscribe" : "01o8vo6no7sd", - "Shadow Publish" : "elztm2jebc1q", - "Shadow Update" : "vuydgrbbbfce" + "MQTT Connect" : "mxn32qkm8npn", + "MQTT Publish" : "gcjhujhhz50p", + "MQTT Subscribe" : "nyiuiwx5yxtj", + "Shadow Publish" : "fttdr8ufljnf", + "Shadow Update" : "ng9t8am2jnry" }, "test_exe_path" : { diff --git a/deviceadvisor/script/DATestRun.py b/deviceadvisor/script/DATestRun.py index 782630bf..38ade00b 100644 --- a/deviceadvisor/script/DATestRun.py +++ b/deviceadvisor/script/DATestRun.py @@ -28,12 +28,21 @@ def process_logs(log_group, log_stream, thing_name): logGroupName=log_group, logStreamName=log_stream ) - log_file = thing_name + ".log" + log_file = "DA_Log_Python_" + thing_name + ".log" f = open(log_file, 'w') for event in response["events"]: f.write(event['message']) f.close() - s3.Bucket(os.environ['DA_S3_NAME']).upload_file(log_file, log_file) + + try: + secrets_client = boto3.client( + "secretsmanager", region_name=os.environ["AWS_DEFAULT_REGION"]) + s3_bucket_name = secrets_client.get_secret_value(SecretId="ci/DeviceAdvisor/s3bucket")["SecretString"] + s3.Bucket(s3_bucket_name).upload_file(log_file, log_file) + print("[Device Advisor] Device Advisor Log file uploaded to "+ log_file) + except Exception: + print ("[Device Advisor] Error: could not store log in S3 bucket!") + os.remove(log_file) print("[Device Advisor] Device Advisor Log file uploaded to "+ log_file) @@ -44,14 +53,24 @@ def sleep_with_backoff(base, max): ############################################## # Initialize variables # create aws clients -client = boto3.client('iot') -dataClient = boto3.client('iot-data') -deviceAdvisor = boto3.client('iotdeviceadvisor') -s3 = boto3.resource('s3') +try: + client = boto3.client('iot', region_name="us-east-1") + dataClient = boto3.client('iot-data', region_name="us-east-1") + deviceAdvisor = boto3.client('iotdeviceadvisor', region_name="us-east-1") + s3 = boto3.resource('s3') +except Exception as ex: + print ("[Device Advisor] Error: could not create boto3 clients.") + print (ex) + exit(-1) # const BACKOFF_BASE = 5 BACKOFF_MAX = 10 +# 60 minutes divided by the maximum back-off = longest time a DA run can last with this script. +MAXIMUM_CYCLE_COUNT = (3600 / BACKOFF_MAX) + +# Did Device Advisor fail a test? If so, this should be true +did_at_least_one_test_fail = False # load test config f = open('deviceadvisor/script/DATestConfig.json') @@ -74,7 +93,7 @@ def sleep_with_backoff(base, max): # Run device advisor for test_name in DATestConfig['tests']: ############################################## - # create a test thing + # create a test thing thing_name = "DATest_" + str(uuid.uuid4()) try: # create_thing_response: @@ -83,27 +102,27 @@ def sleep_with_backoff(base, max): # 'thingArn': 'string', # 'thingId': 'string' # } - print("[Device Advisor]Info: Started to create thing...") + print("[Device Advisor] Info: Started to create thing...") create_thing_response = client.create_thing( thingName=thing_name ) os.environ["DA_THING_NAME"] = thing_name - + except Exception as e: - print("[Device Advisor]Error: Failed to create thing: " + thing_name) + print("[Device Advisor] Error: Failed to create thing: " + thing_name) exit(-1) ############################################## # create certificate and keys used for testing try: - print("[Device Advisor]Info: Started to create certificate...") + print("[Device Advisor] Info: Started to create certificate...") # create_cert_response: # { # 'certificateArn': 'string', # 'certificateId': 'string', # 'certificatePem': 'string', - # 'keyPair': + # 'keyPair': # { # 'PublicKey': 'string', # 'PrivateKey': 'string' @@ -121,39 +140,58 @@ def sleep_with_backoff(base, max): f = open(key_path, "w") f.write(create_cert_response['keyPair']['PrivateKey']) f.close() - - # setup environment variable + + # setup environment variable os.environ["DA_CERTI"] = certificate_path os.environ["DA_KEY"] = key_path - except: - client.delete_thing(thingName = thing_name) - print("[Device Advisor]Error: Failed to create certificate.") + except Exception: + try: + client.delete_thing(thingName = thing_name) + except Exception: + print("[Device Advisor] Error: Could not delete thing.") + print("[Device Advisor] Error: Failed to create certificate.") + exit(-1) + + certificate_arn = create_cert_response['certificateArn'] + certificate_id = create_cert_response['certificateId'] + + ############################################## + # attach policy to certificate + try: + secrets_client = boto3.client( + "secretsmanager", region_name=os.environ["AWS_DEFAULT_REGION"]) + policy_name = secrets_client.get_secret_value(SecretId="ci/DeviceAdvisor/policy_name")["SecretString"] + client.attach_policy ( + policyName= policy_name, + target = certificate_arn + ) + except Exception as ex: + print (ex) + delete_thing_with_certi(thing_name, certificate_id, certificate_arn ) + print("[Device Advisor] Error: Failed to attach policy.") exit(-1) ############################################## # attach certification to thing try: - print("[Device Advisor]Info: Attach certificate to test thing...") + print("[Device Advisor] Info: Attach certificate to test thing...") # attache the certificate to thing client.attach_thing_principal( thingName = thing_name, - principal = create_cert_response['certificateArn'] + principal = certificate_arn ) - certificate_arn = create_cert_response['certificateArn'] - certificate_id = create_cert_response['certificateId'] - - except: + except Exception: delete_thing_with_certi(thing_name, certificate_id ,certificate_arn ) - print("[Device Advisor]Error: Failed to attach certificate.") + print("[Device Advisor] Error: Failed to attach certificate.") exit(-1) - try: ###################################### # set default shadow, for shadow update, if the # shadow does not exists, update will fail + print("[Device Advisor] Info: About to update shadow.") payload_shadow = json.dumps( { "state": { @@ -170,9 +208,10 @@ def sleep_with_backoff(base, max): payload = payload_shadow) get_shadow_response = dataClient.get_thing_shadow(thingName = thing_name) # make sure shadow is created before we go to next step - while(get_shadow_response is None): + print("[Device Advisor] Info: About to wait for shadow update.") + while(get_shadow_response is None): get_shadow_response = dataClient.get_thing_shadow(thingName = thing_name) - + # start device advisor test # test_start_response # { @@ -180,40 +219,50 @@ def sleep_with_backoff(base, max): # 'suiteRunArn': 'string', # 'createdAt': datetime(2015, 1, 1) # } - print("[Device Advisor]Info: Start device advisor test: " + test_name) + print("[Device Advisor] Info: Start device advisor test: " + test_name) sleep_with_backoff(BACKOFF_BASE, BACKOFF_MAX) test_start_response = deviceAdvisor.start_suite_run( - suiteDefinitionId=DATestConfig['test_suite_ids'][test_name], - suiteRunConfiguration={ - 'primaryDevice': { - 'thingArn': create_thing_response['thingArn'], - }, - 'parallelRun': True + suiteDefinitionId=DATestConfig['test_suite_ids'][test_name], + suiteRunConfiguration={ + 'primaryDevice': { + 'thingArn': create_thing_response['thingArn'], + }, + 'parallelRun': True }) # get DA endpoint + print("[Device Advisor] Info: Getting Device Advisor endpoint.") endpoint_response = deviceAdvisor.get_endpoint( thingArn = create_thing_response['thingArn'] ) os.environ['DA_ENDPOINT'] = endpoint_response['endpoint'] + cycle_number = 0 while True: + cycle_number += 1 + if (cycle_number >= MAXIMUM_CYCLE_COUNT): + print(f"[Device Advisor] Error: {cycle_number} of cycles lasting {BACKOFF_BASE} to {BACKOFF_MAX} seconds have passed.") + raise Exception(f"ERROR - {cycle_number} of cycles lasting {BACKOFF_BASE} to {BACKOFF_MAX} seconds have passed.") + # Add backoff to avoid TooManyRequestsException sleep_with_backoff(BACKOFF_BASE, BACKOFF_MAX) + print ("[Device Advisor] Info: About to get Device Advisor suite run.") test_result_responds = deviceAdvisor.get_suite_run( suiteDefinitionId=DATestConfig['test_suite_ids'][test_name], suiteRunId=test_start_response['suiteRunId'] ) + # If the status is PENDING or the responds does not loaded, the test suite is still loading if (test_result_responds['status'] == 'PENDING' or len(test_result_responds['testResult']['groups']) == 0 or # test group has not been loaded len(test_result_responds['testResult']['groups'][0]['tests']) == 0 or #test case has not been loaded test_result_responds['testResult']['groups'][0]['tests'][0]['status'] == 'PENDING'): continue - + # Start to run the test sample after the status turns into RUNNING - elif (test_result_responds['status'] == 'RUNNING' and + elif (test_result_responds['status'] == 'RUNNING' and test_result_responds['testResult']['groups'][0]['tests'][0]['status'] == 'RUNNING'): + print ("[Device Advisor] Info: About to get start Device Advisor companion test application.") exe_path = os.path.join("deviceadvisor/tests/",DATestConfig['test_exe_path'][test_name]) result = subprocess.run('python3 ' + exe_path, timeout = 60*2, shell = True) # If the test finalizing then store the test result @@ -221,6 +270,7 @@ def sleep_with_backoff(base, max): test_result[test_name] = test_result_responds['status'] # If the test failed, upload the logs to S3 before clean up if(test_result[test_name] != "PASS"): + print ("[Device Advisor] Info: About to upload log to S3.") log_url = test_result_responds['testResult']['groups'][0]['tests'][0]['logUrl'] group_string = re.search('group=(.*);', log_url) log_group = group_string.group(1) @@ -229,9 +279,10 @@ def sleep_with_backoff(base, max): process_logs(log_group, log_stream, thing_name) delete_thing_with_certi(thing_name, certificate_id ,certificate_arn) break - except Exception as e: + except Exception: delete_thing_with_certi(thing_name, certificate_id ,certificate_arn) - print("[Device Advisor]Error: Failed to test: "+ test_name + e) + print("[Device Advisor] Error: Failed to test: "+ test_name) + did_at_least_one_test_fail = True exit(-1) ############################################## @@ -241,10 +292,14 @@ def sleep_with_backoff(base, max): for test in test_result: if(test_result[test] != "PASS" and test_result[test] != "PASS_WITH_WARNINGS"): - print("[Device Advisor]Error: Test \"" + test + "\" Failed with status:" + test_result[test]) + print("[Device Advisor] Error: Test \"" + test + "\" Failed with status:" + test_result[test]) failed = True if failed: # if the test failed, we dont clean the Thing so that we can track the error exit(-1) +if (did_at_least_one_test_fail == True): + print("[Device Advisor] Error: At least one test failed!") + exit(-1) + exit(0) diff --git a/deviceadvisor/tests/da_test_utils.py b/deviceadvisor/tests/da_test_utils.py index 1de0980b..df26e2df 100644 --- a/deviceadvisor/tests/da_test_utils.py +++ b/deviceadvisor/tests/da_test_utils.py @@ -6,8 +6,8 @@ class TestType(Enum): CONNECT = 1 - SUB_PUB = 1 - SHADOW = 1 + SUB_PUB = 2 + SHADOW = 3 class DATestUtils: endpoint = os.getenv('DA_ENDPOINT') @@ -17,7 +17,6 @@ class DATestUtils: thing_name = os.getenv('DA_THING_NAME') shadowProperty = os.getenv('DA_SHADOW_PROPERTY') shadowValue = os.getenv('DA_SHADOW_VALUE_SET') - client_id = "test-" + str(uuid4()) @classmethod def valid(cls, test_type): @@ -29,5 +28,9 @@ def valid(cls, test_type): if (not (cls.thing_name and cls.shadowProperty and cls.shadowValue) and test_type == TestType.SHADOW): return False - + return True + + @classmethod + def generate_client_id(_self, postfix): + return "test-DA" + str(uuid4()) + postfix diff --git a/deviceadvisor/tests/mqtt_connect.py b/deviceadvisor/tests/mqtt_connect.py index a7c5cb0b..0813f676 100644 --- a/deviceadvisor/tests/mqtt_connect.py +++ b/deviceadvisor/tests/mqtt_connect.py @@ -1,6 +1,5 @@ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0. - from awsiot import mqtt_connection_builder from da_test_utils import DATestUtils, TestType @@ -13,13 +12,14 @@ quit(-1) mqtt_connection = mqtt_connection_builder.mtls_from_path( - endpoint=DATestUtils.endpoint, - cert_filepath=DATestUtils.certificatePath, - pri_key_filepath=DATestUtils.keyPath, - client_id = DATestUtils.client_id, - clean_session = True, - ping_timeout_ms = 6000) - + endpoint = DATestUtils.endpoint, + cert_filepath = DATestUtils.certificatePath, + pri_key_filepath = DATestUtils.keyPath, + client_id = DATestUtils.generate_client_id("-connect"), + clean_session = True, + tcp_connect_timeout_ms = 60000, # 1 minute + keep_alive_secs = 60000, # 1 minute + ping_timeout_ms = 120000) # 2 minutes connect_future = mqtt_connection.connect() # Future.result() waits until a result is available diff --git a/deviceadvisor/tests/mqtt_publish.py b/deviceadvisor/tests/mqtt_publish.py index 7ffbb116..976a367a 100644 --- a/deviceadvisor/tests/mqtt_publish.py +++ b/deviceadvisor/tests/mqtt_publish.py @@ -12,12 +12,14 @@ quit(-1) mqtt_connection = mqtt_connection_builder.mtls_from_path( - endpoint=DATestUtils.endpoint, - cert_filepath=DATestUtils.certificatePath, - pri_key_filepath=DATestUtils.keyPath, - client_id=DATestUtils.client_id, + endpoint = DATestUtils.endpoint, + cert_filepath = DATestUtils.certificatePath, + pri_key_filepath = DATestUtils.keyPath, + client_id = DATestUtils.generate_client_id("-pub"), clean_session = True, - ping_timeout_ms = 6000) + tcp_connect_timeout_ms = 60000, # 1 minute + keep_alive_secs = 60000, # 1 minute + ping_timeout_ms = 120000) # 2 minutes connect_future = mqtt_connection.connect() # Future.result() waits until a result is available @@ -25,7 +27,7 @@ message = "Hello World" message_json = json.dumps(message) - # Device advisor test will not return PUBACK, therefore we use AT_MOST_ONCE so that + # Device advisor test will not return PUBACK, therefore we use AT_MOST_ONCE so that # we dont busy wait for PUBACK publish_future, packet_id = mqtt_connection.publish( topic=DATestUtils.topic, diff --git a/deviceadvisor/tests/mqtt_subscribe.py b/deviceadvisor/tests/mqtt_subscribe.py index 09bdd4d6..f38cc585 100644 --- a/deviceadvisor/tests/mqtt_subscribe.py +++ b/deviceadvisor/tests/mqtt_subscribe.py @@ -12,13 +12,14 @@ quit(-1) mqtt_connection = mqtt_connection_builder.mtls_from_path( - endpoint=DATestUtils.endpoint, - cert_filepath=DATestUtils.certificatePath, - pri_key_filepath=DATestUtils.keyPath, - client_id = DATestUtils.client_id, + endpoint = DATestUtils.endpoint, + cert_filepath = DATestUtils.certificatePath, + pri_key_filepath = DATestUtils.keyPath, + client_id = DATestUtils.generate_client_id("-sub"), clean_session = True, - ping_timeout_ms = 6000) - + tcp_connect_timeout_ms = 60000, # 1 minute + keep_alive_secs = 60000, # 1 minute + ping_timeout_ms = 120000) # 2 minutes connect_future = mqtt_connection.connect() # Future.result() waits until a result is available @@ -29,7 +30,7 @@ topic=DATestUtils.topic, qos=mqtt.QoS.AT_MOST_ONCE) subscribe_future.result() - + # Disconnect disconnect_future = mqtt_connection.disconnect() disconnect_future.result() diff --git a/deviceadvisor/tests/shadow_update.py b/deviceadvisor/tests/shadow_update.py index 8d417444..abe569d6 100644 --- a/deviceadvisor/tests/shadow_update.py +++ b/deviceadvisor/tests/shadow_update.py @@ -14,18 +14,19 @@ quit(-1) mqtt_connection = mqtt_connection_builder.mtls_from_path( - endpoint=DATestUtils.endpoint, - cert_filepath=DATestUtils.certificatePath, - pri_key_filepath=DATestUtils.keyPath, - client_id = DATestUtils.client_id, + endpoint = DATestUtils.endpoint, + cert_filepath = DATestUtils.certificatePath, + pri_key_filepath = DATestUtils.keyPath, + client_id = DATestUtils.generate_client_id("-shadow"), clean_session = True, - ping_timeout_ms = 6000) + tcp_connect_timeout_ms = 60000, # 1 minute + keep_alive_secs = 60000, # 1 minute + ping_timeout_ms = 120000) # 2 minutes connect_future = mqtt_connection.connect() connect_future.result() shadow_client = iotshadow.IotShadowClient(mqtt_connection) - # Publish shadow value request = iotshadow.UpdateShadowRequest( thing_name=DATestUtils.thing_name, @@ -34,7 +35,7 @@ desired={ DATestUtils.shadowProperty: DATestUtils.shadowValue }, ) ) - # Device advisor test will not return PUBACK, therefore we use AT_MOST_ONCE so that + # Device advisor test will not return PUBACK, therefore we use AT_MOST_ONCE so that # we dont busy wait for PUBACK shadow_future = shadow_client.publish_update_shadow(request, mqtt.QoS.AT_MOST_ONCE) shadow_future.result() diff --git a/samples/README.md b/samples/README.md index 03007c6a..902ce823 100644 --- a/samples/README.md +++ b/samples/README.md @@ -18,6 +18,7 @@ First, install the aws-iot-devices-sdk-python-v2 with following the instructions Then change into the samples directory to run the Python commands to execute the samples. You can view the commands of a sample like this: ``` sh +# For Windows: replace 'python3' with 'python' python3 pubsub.py --help ``` @@ -77,6 +78,7 @@ Your Thing's [Policy](https://docs.aws.amazon.com/iot/latest/developerguide/iot- Run the sample like this: ``` sh +# For Windows: replace 'python3' with 'python' python3 pubsub.py --endpoint --ca_file --cert --key ``` @@ -111,6 +113,7 @@ Your Thing's [Policy](https://docs.aws.amazon.com/iot/latest/developerguide/iot- Run the sample like this: ``` sh +# For Windows: replace 'python3' with 'python' python3 basic_connect.py --endpoint --ca_file --cert --key ``` @@ -145,6 +148,7 @@ Your Thing's [Policy](https://docs.aws.amazon.com/iot/latest/developerguide/iot- Run the sample like this: ``` sh +# For Windows: replace 'python3' with 'python' python3 websocket_connect.py --endpoint --ca_file --signing_region ``` @@ -222,6 +226,7 @@ To run this sample using [SoftHSM2](https://www.opendnssec.org/softhsm/) as the 5) Now you can run the sample: ```sh + # For Windows: replace 'python3' with 'python' python3 pkcs11_connect.py --endpoint --ca_file --cert --pkcs11_lib --pin --token_label --key_label ``` @@ -307,6 +312,7 @@ To run this sample with a basic certificate from AWS IoT Core: 4) Now you can run the sample: ```sh + # For Windows: replace 'python3' with 'python' python3 windows_cert_connect.py --endpoint --ca_file --cert ``` @@ -338,6 +344,7 @@ Your Thing's [Policy](https://docs.aws.amazon.com/iot/latest/developerguide/iot- Run the sample like this: ``` sh +# For Windows: replace 'python3' with 'python' python3 custom_authorizer_connect.py --endpoint --ca_file --custom_auth_authorizer_name ``` @@ -367,6 +374,7 @@ Source: `samples/shadow.py` Run the sample like this: ``` sh +# For Windows: replace 'python3' with 'python' python3 shadow.py --endpoint --ca_file --cert --key --thing-name ``` @@ -428,14 +436,16 @@ Your Thing's [Policy](https://docs.aws.amazon.com/iot/latest/developerguide/iot- This sample uses the AWS IoT [Jobs](https://docs.aws.amazon.com/iot/latest/developerguide/iot-jobs.html) -Service to receive and execute operations -on the device. Imagine periodic software updates that must be sent to and +Service to get a list of pending jobs and +then execution operations on these pending jobs until there are no more +remaining on the device. Imagine periodic software updates that must be sent to and executed on devices in the wild. This sample requires you to create jobs for your device to execute. See [instructions here](https://docs.aws.amazon.com/iot/latest/developerguide/create-manage-jobs.html). -On startup, the sample tries to start the next pending job execution. +On startup, the sample tries to get a list of all the in-progress and queued +jobs and display them in a list. Then it tries to start the next pending job execution. If such a job exists, the sample emulates "doing work" by spawning a thread that sleeps for several seconds before marking the job as SUCCEEDED. When no pending job executions exist, the sample sits in an idle state. @@ -450,51 +460,48 @@ Source: `samples/jobs.py` Run the sample like this: ``` sh +# For Windows: replace 'python3' with 'python' python3 jobs.py --endpoint --ca_file --cert --key --thing_name ``` Your Thing's [Policy](https://docs.aws.amazon.com/iot/latest/developerguide/iot-policies.html) must provide privileges for this sample to connect, subscribe, publish, and receive. Make sure your policy allows a client ID of `test-*` to connect or use `--client_id ` to send the client ID your policy supports.
-(see sample policy) +Sample Policy 
 {
   "Version": "2012-10-17",
   "Statement": [
     {
       "Effect": "Allow",
-      "Action": [
-        "iot:Publish"
-      ],
+      "Action": "iot:Publish",
       "Resource": [
         "arn:aws:iot:region:account:topic/$aws/things/thingname/jobs/start-next",
-        "arn:aws:iot:region:account:topic/$aws/things/thingname/jobs/*/update"
+        "arn:aws:iot:region:account:topic/$aws/things/thingname/jobs/*/update",
+        "arn:aws:iot:region:account:topic/$aws/things/thingname/jobs/*/get",
+        "arn:aws:iot:region:account:topic/$aws/things/thingname/jobs/get"
       ]
     },
     {
       "Effect": "Allow",
-      "Action": [
-        "iot:Receive"
-      ],
+      "Action": "iot:Receive",
       "Resource": [
         "arn:aws:iot:region:account:topic/$aws/things/thingname/jobs/notify-next",
-        "arn:aws:iot:region:account:topic/$aws/things/thingname/jobs/start-next/accepted",
-        "arn:aws:iot:region:account:topic/$aws/things/thingname/jobs/start-next/rejected",
-        "arn:aws:iot:region:account:topic/$aws/things/thingname/jobs/*/update/accepted",
-        "arn:aws:iot:region:account:topic/$aws/things/thingname/jobs/*/update/rejected"
+        "arn:aws:iot:region:account:topic/$aws/things/thingname/jobs/start-next/*",
+        "arn:aws:iot:region:account:topic/$aws/things/thingname/jobs/*/update/*",
+        "arn:aws:iot:region:account:topic/$aws/things/thingname/jobs/get/*",
+        "arn:aws:iot:region:account:topic/$aws/things/thingname/jobs/*/get/*"
       ]
     },
     {
       "Effect": "Allow",
-      "Action": [
-        "iot:Subscribe"
-      ],
+      "Action": "iot:Subscribe",
       "Resource": [
         "arn:aws:iot:region:account:topicfilter/$aws/things/thingname/jobs/notify-next",
-        "arn:aws:iot:region:account:topicfilter/$aws/things/thingname/jobs/start-next/accepted",
-        "arn:aws:iot:region:account:topicfilter/$aws/things/thingname/jobs/start-next/rejected",
-        "arn:aws:iot:region:account:topicfilter/$aws/things/thingname/jobs/*/update/accepted",
-        "arn:aws:iot:region:account:topicfilter/$aws/things/thingname/jobs/*/update/rejected"
+        "arn:aws:iot:region:account:topicfilter/$aws/things/thingname/jobs/start-next/*",
+        "arn:aws:iot:region:account:topicfilter/$aws/things/thingname/jobs/*/update/*",
+        "arn:aws:iot:region:account:topicfilter/$aws/things/thingname/jobs/get/*",
+        "arn:aws:iot:region:account:topicfilter/$aws/things/thingname/jobs/*/get/*"
       ]
     },
     {
@@ -511,7 +518,7 @@ Your Thing's [Policy](https://docs.aws.amazon.com/iot/latest/developerguide/iot-
 
 This sample uses the AWS IoT
 [Fleet provisioning](https://docs.aws.amazon.com/iot/latest/developerguide/provision-wo-cert.html)
-to provision devices using either a CSR or KeysAndcertificate and subsequently calls RegisterThing.
+to provision devices using either a CSR or Keys-And-Certificate and subsequently calls RegisterThing.
 
 On startup, the script subscribes to topics based on the request type of either CSR or Keys topics,
 publishes the request to corresponding topic and calls RegisterThing.
@@ -520,11 +527,13 @@ Source: `samples/fleetprovisioning.py`
 
 Run the sample using createKeysAndCertificate:
 ``` sh
+# For Windows: replace 'python3' with 'python'
 python3 fleetprovisioning.py --endpoint  --ca_file  --cert  --key  --template_name  --template_parameters 
 ```
 
 Run the sample using createCertificateFromCsr:
 ``` sh
+# For Windows: replace 'python3' with 'python'
 python3 fleetprovisioning.py --endpoint  --ca_file  --cert  --key  --template_name  --template_parameters  --csr 
 ```
 
@@ -538,9 +547,7 @@ Your Thing's [Policy](https://docs.aws.amazon.com/iot/latest/developerguide/iot-
   "Statement": [
     {
       "Effect": "Allow",
-      "Action": [
-        "iot:Publish"
-      ],
+      "Action": "iot:Publish",
       "Resource": [
         "arn:aws:iot:region:account:topic/$aws/certificates/create/json",
         "arn:aws:iot:region:account:topic/$aws/certificates/create-from-csr/json",
@@ -550,8 +557,7 @@ Your Thing's [Policy](https://docs.aws.amazon.com/iot/latest/developerguide/iot-
     {
       "Effect": "Allow",
       "Action": [
-        "iot:Receive",
-        "iot:Subscribe"
+        "iot:Receive"
       ],
       "Resource": [
         "arn:aws:iot:region:account:topic/$aws/certificates/create/json/accepted",
@@ -562,6 +568,20 @@ Your Thing's [Policy](https://docs.aws.amazon.com/iot/latest/developerguide/iot-
         "arn:aws:iot:region:account:topic/$aws/provisioning-templates/templatename/provision/json/rejected"
       ]
     },
+    {
+      "Effect": "Allow",
+      "Action": [
+        "iot:Subscribe"
+      ],
+      "Resource": [
+        "arn:aws:iot:region:account:topicfilter/$aws/certificates/create/json/accepted",
+        "arn:aws:iot:region:account:topicfilter/$aws/certificates/create/json/rejected",
+        "arn:aws:iot:region:account:topicfilter/$aws/certificates/create-from-csr/json/accepted",
+        "arn:aws:iot:region:account:topicfilter/$aws/certificates/create-from-csr/json/rejected",
+        "arn:aws:iot:region:account:topicfilter/$aws/provisioning-templates/templatename/provision/json/accepted",
+        "arn:aws:iot:region:account:topicfilter/$aws/provisioning-templates/templatename/provision/json/rejected"
+      ]
+    },
     {
       "Effect": "Allow",
       "Action": "iot:Connect",
@@ -606,9 +626,104 @@ aws iot create-provisioning-template \
         --enabled
 ```
 The rest of the instructions assume you have used the following for the template body:
+
+

+(see template body)
 ``` sh
-{\"Parameters\":{\"DeviceLocation\":{\"Type\":\"String\"},\"AWS::IoT::Certificate::Id\":{\"Type\":\"String\"},\"SerialNumber\":{\"Type\":\"String\"}},\"Mappings\":{\"LocationTable\":{\"Seattle\":{\"LocationUrl\":\"https://example.aws\"}}},\"Resources\":{\"thing\":{\"Type\":\"AWS::IoT::Thing\",\"Properties\":{\"ThingName\":{\"Fn::Join\":[\"\",[\"ThingPrefix_\",{\"Ref\":\"SerialNumber\"}]]},\"AttributePayload\":{\"version\":\"v1\",\"serialNumber\":\"serialNumber\"}},\"OverrideSettings\":{\"AttributePayload\":\"MERGE\",\"ThingTypeName\":\"REPLACE\",\"ThingGroups\":\"DO_NOTHING\"}},\"certificate\":{\"Type\":\"AWS::IoT::Certificate\",\"Properties\":{\"CertificateId\":{\"Ref\":\"AWS::IoT::Certificate::Id\"},\"Status\":\"Active\"},\"OverrideSettings\":{\"Status\":\"REPLACE\"}},\"policy\":{\"Type\":\"AWS::IoT::Policy\",\"Properties\":{\"PolicyDocument\":{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":[\"iot:Connect\",\"iot:Subscribe\",\"iot:Publish\",\"iot:Receive\"],\"Resource\":\"*\"}]}}}},\"DeviceConfiguration\":{\"FallbackUrl\":\"https://www.example.com/test-site\",\"LocationUrl\":{\"Fn::FindInMap\":[\"LocationTable\",{\"Ref\":\"DeviceLocation\"},\"LocationUrl\"]}}}
+{
+  "Parameters": {
+    "DeviceLocation": {
+      "Type": "String"
+    },
+    "AWS::IoT::Certificate::Id": {
+      "Type": "String"
+    },
+    "SerialNumber": {
+      "Type": "String"
+    }
+  },
+  "Mappings": {
+    "LocationTable": {
+      "Seattle": {
+        "LocationUrl": "https://example.aws"
+      }
+    }
+  },
+  "Resources": {
+    "thing": {
+      "Type": "AWS::IoT::Thing",
+      "Properties": {
+        "ThingName": {
+          "Fn::Join": [
+            "",
+            [
+              "ThingPrefix_",
+              {
+                "Ref": "SerialNumber"
+              }
+            ]
+          ]
+        },
+        "AttributePayload": {
+          "version": "v1",
+          "serialNumber": "serialNumber"
+        }
+      },
+      "OverrideSettings": {
+        "AttributePayload": "MERGE",
+        "ThingTypeName": "REPLACE",
+        "ThingGroups": "DO_NOTHING"
+      }
+    },
+    "certificate": {
+      "Type": "AWS::IoT::Certificate",
+      "Properties": {
+        "CertificateId": {
+          "Ref": "AWS::IoT::Certificate::Id"
+        },
+        "Status": "Active"
+      },
+      "OverrideSettings": {
+        "Status": "REPLACE"
+      }
+    },
+    "policy": {
+      "Type": "AWS::IoT::Policy",
+      "Properties": {
+        "PolicyDocument": {
+          "Version": "2012-10-17",
+          "Statement": [
+            {
+              "Effect": "Allow",
+              "Action": [
+                "iot:Connect",
+                "iot:Subscribe",
+                "iot:Publish",
+                "iot:Receive"
+              ],
+              "Resource": "*"
+            }
+          ]
+        }
+      }
+    }
+  },
+  "DeviceConfiguration": {
+    "FallbackUrl": "https://www.example.com/test-site",
+    "LocationUrl": {
+      "Fn::FindInMap": [
+        "LocationTable",
+        {
+          "Ref": "DeviceLocation"
+        },
+        "LocationUrl"
+      ]
+    }
+  }
+}
 ```
+

+
 If you use a different body, you may need to pass in different template parameters.
 
 #### Running the sample and provisioning using a certificate-key set from a provisioning claim
@@ -636,6 +751,7 @@ to perform the actual provisioning. If you are not using the temporary provision
 and `--key` appropriately:
 
 ``` sh
+# For Windows: replace 'python3' with 'python'
 python3 fleetprovisioning.py \
         --endpoint  \
         --ca_file  \
@@ -676,6 +792,7 @@ Finally, supply the certificate signing request while invoking the provisioning
 using a permanent certificate set, replace the paths specified in the `--cert` and `--key` arguments:
 
 ``` sh
+# For Windows: replace 'python3' with 'python'
 python3 fleetprovisioning.py \
         --endpoint  \
         --ca_file  \
diff --git a/samples/basic_connect.py b/samples/basic_connect.py
index a3761c0b..28cb09ab 100644
--- a/samples/basic_connect.py
+++ b/samples/basic_connect.py
@@ -21,8 +21,10 @@
 cmdUtils.register_command("client_id", "",
                           "Client ID to use for MQTT connection (optional, default='test-*').",
                           default="test-" + str(uuid4()))
+cmdUtils.register_command("is_ci", "", "If present the sample will run in CI mode (optional, default='None')")
 # Needs to be called so the command utils parse the commands
 cmdUtils.get_args()
+is_ci = cmdUtils.get_command("is_ci", None) != None
 
 # Callback when connection is accidentally lost.
 def on_connection_interrupted(connection, error, **kwargs):
@@ -39,8 +41,11 @@ def on_connection_resumed(connection, return_code, session_present, **kwargs):
     # (see build_direct_mqtt_connection for implementation)
     mqtt_connection = cmdUtils.build_direct_mqtt_connection(on_connection_interrupted, on_connection_resumed)
 
-    print("Connecting to {} with client ID '{}'...".format(
-        cmdUtils.get_command(cmdUtils.m_cmd_endpoint), cmdUtils.get_command("client_id")))
+    if is_ci == False:
+        print("Connecting to {} with client ID '{}'...".format(
+            cmdUtils.get_command(cmdUtils.m_cmd_endpoint), cmdUtils.get_command("client_id")))
+    else:
+        print("Connecting to endpoint with client ID")
 
     connect_future = mqtt_connection.connect()
 
diff --git a/samples/custom_authorizer_connect.py b/samples/custom_authorizer_connect.py
index 959bac24..93ac1c62 100644
--- a/samples/custom_authorizer_connect.py
+++ b/samples/custom_authorizer_connect.py
@@ -17,8 +17,10 @@
 cmdUtils.register_command("client_id", "",
                           "Client ID to use for MQTT connection (optional, default='test-*').",
                           default="test-" + str(uuid4()))
+cmdUtils.register_command("is_ci", "", "If present the sample will run in CI mode (optional, default='None')")
 # Needs to be called so the command utils parse the commands
 cmdUtils.get_args()
+is_ci = cmdUtils.get_command("is_ci", None) != None
 
 
 def on_connection_interrupted(connection, error, **kwargs):
@@ -47,8 +49,11 @@ def on_connection_resumed(connection, return_code, session_present, **kwargs):
         clean_session=False,
         keep_alive_secs=30)
 
-    print("Connecting to {} with client ID '{}'...".format(
-        cmdUtils.get_command(cmdUtils.m_cmd_endpoint), cmdUtils.get_command("client_id")))
+    if is_ci == False:
+        print("Connecting to {} with client ID '{}'...".format(
+            cmdUtils.get_command(cmdUtils.m_cmd_endpoint), cmdUtils.get_command("client_id")))
+    else:
+        print("Connecting to endpoint with client ID")
 
     connect_future = mqtt_connection.connect()
 
diff --git a/samples/fleetprovisioning.py b/samples/fleetprovisioning.py
index c20a5aa7..92bddb39 100644
--- a/samples/fleetprovisioning.py
+++ b/samples/fleetprovisioning.py
@@ -37,6 +37,7 @@
 cmdUtils.register_command("csr", "", "Path to CSR in Pem format (optional).")
 cmdUtils.register_command("template_name", "", "The name of your provisioning template.")
 cmdUtils.register_command("template_parameters", "", "Template parameters json.")
+cmdUtils.register_command("is_ci", "", "If present the sample will run in CI mode (optional, default='None')")
 # Needs to be called so the command utils parse the commands
 cmdUtils.get_args()
 
@@ -47,6 +48,7 @@
 createKeysAndCertificateResponse = None
 createCertificateFromCsrResponse = None
 registerThingResponse = None
+is_ci = cmdUtils.get_command("is_ci", None) != None
 
 class LockedData:
     def __init__(self):
@@ -112,7 +114,8 @@ def createkeysandcertificate_execution_accepted(response):
     try:
         global createKeysAndCertificateResponse
         createKeysAndCertificateResponse = response
-        print("Received a new message {}".format(createKeysAndCertificateResponse))
+        if (is_ci == False):
+            print("Received a new message {}".format(createKeysAndCertificateResponse))
 
         return
 
@@ -129,7 +132,8 @@ def createcertificatefromcsr_execution_accepted(response):
     try:
         global createCertificateFromCsrResponse
         createCertificateFromCsrResponse = response
-        print("Received a new message {}".format(createCertificateFromCsrResponse))
+        if (is_ci == False):
+            print("Received a new message {}".format(createCertificateFromCsrResponse))
         global certificateOwnershipToken
         certificateOwnershipToken = response.certificate_ownership_token
 
@@ -148,7 +152,8 @@ def registerthing_execution_accepted(response):
     try:
         global registerThingResponse
         registerThingResponse = response
-        print("Received a new message {} ".format(registerThingResponse))
+        if (is_ci == False):
+            print("Received a new message {} ".format(registerThingResponse))
         return
 
     except Exception as e:
@@ -190,7 +195,10 @@ def waitForCreateKeysAndCertificateResponse():
     while loopCount < 10 and createKeysAndCertificateResponse is None:
         if createKeysAndCertificateResponse is not None:
             break
-        print('Waiting... CreateKeysAndCertificateResponse: ' + json.dumps(createKeysAndCertificateResponse))
+        if is_ci == False:
+            print('Waiting... CreateKeysAndCertificateResponse: ' + json.dumps(createKeysAndCertificateResponse))
+        else:
+            print("Waiting... CreateKeysAndCertificateResponse: ...")
         loopCount += 1
         time.sleep(1)
 
@@ -200,7 +208,10 @@ def waitForCreateCertificateFromCsrResponse():
     while loopCount < 10 and createCertificateFromCsrResponse is None:
         if createCertificateFromCsrResponse is not None:
             break
-        print('Waiting...CreateCertificateFromCsrResponse: ' + json.dumps(createCertificateFromCsrResponse))
+        if is_ci == False:
+            print('Waiting...CreateCertificateFromCsrResponse: ' + json.dumps(createCertificateFromCsrResponse))
+        else:
+            print("Waiting... CreateCertificateFromCsrResponse: ...")
         loopCount += 1
         time.sleep(1)
 
@@ -211,15 +222,21 @@ def waitForRegisterThingResponse():
         if registerThingResponse is not None:
             break
         loopCount += 1
-        print('Waiting... RegisterThingResponse: ' + json.dumps(registerThingResponse))
+        if is_ci == False:
+            print('Waiting... RegisterThingResponse: ' + json.dumps(registerThingResponse))
+        else:
+            print('Waiting... RegisterThingResponse: ...')
         time.sleep(1)
 
 if __name__ == '__main__':
     proxy_options = None
     mqtt_connection = cmdUtils.build_mqtt_connection(on_connection_interrupted, on_connection_resumed)
 
-    print("Connecting to {} with client ID '{}'...".format(
-        cmdUtils.get_command(cmdUtils.m_cmd_endpoint), cmdUtils.get_command("client_id")))
+    if is_ci == False:
+        print("Connecting to {} with client ID '{}'...".format(
+            cmdUtils.get_command(cmdUtils.m_cmd_endpoint), cmdUtils.get_command("client_id")))
+    else:
+        print("Connecting to endpoint with client ID")
 
     connected_future = mqtt_connection.connect()
 
diff --git a/samples/jobs.py b/samples/jobs.py
index c1ca0e3a..239bb8d8 100644
--- a/samples/jobs.py
+++ b/samples/jobs.py
@@ -1,20 +1,20 @@
 # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 # SPDX-License-Identifier: Apache-2.0.
 
-import argparse
-from awscrt import auth, http, io, mqtt
+from awscrt import mqtt
 from awsiot import iotjobs
-from awsiot import mqtt_connection_builder
 from concurrent.futures import Future
 import sys
 import threading
 import time
 import traceback
+import time
 from uuid import uuid4
 
 # - Overview -
-# This sample uses the AWS IoT Jobs Service to receive and execute operations
-# on the device. Imagine periodic software updates that must be sent to and
+# This sample uses the AWS IoT Jobs Service to get a list of pending jobs and
+# then execution operations on these pending jobs until there are no more
+# remaining on the device. Imagine periodic software updates that must be sent to and
 # executed on devices in the wild.
 #
 # - Instructions -
@@ -22,7 +22,8 @@
 # https://docs.aws.amazon.com/iot/latest/developerguide/create-manage-jobs.html
 #
 # - Detail -
-# On startup, the sample tries to start the next pending job execution.
+# On startup, the sample tries to get a list of all the in-progress and queued
+# jobs and display them in a list. Then it tries to start the next pending job execution.
 # If such a job exists, the sample emulates "doing work" by spawning a thread
 # that sleeps for several seconds before marking the job as SUCCEEDED. When no
 # pending job executions exist, the sample sits in an idle state.
@@ -48,12 +49,14 @@
 cmdUtils.register_command("port", "", "Connection port. AWS IoT supports 443 and 8883 (optional, default=auto).", type=int)
 cmdUtils.register_command("thing_name", "", "The name assigned to your IoT Thing", required=True)
 cmdUtils.register_command("job_time", "", "Emulate working on a job by sleeping this many seconds (optional, default='5')", default=5, type=int)
+cmdUtils.register_command("is_ci", "", "If present the sample will run in CI mode (optional, default='None'. Will just describe job if set)")
 # Needs to be called so the command utils parse the commands
 cmdUtils.get_args()
 
 mqtt_connection = None
 jobs_client = None
 jobs_thing_name = cmdUtils.get_command_required("thing_name")
+is_ci = cmdUtils.get_command("is_ci", None) != None
 
 class LockedData:
     def __init__(self):
@@ -61,6 +64,7 @@ def __init__(self):
         self.disconnect_called = False
         self.is_working_on_job = False
         self.is_next_job_waiting = False
+        self.got_job_response = False
 
 locked_data = LockedData()
 
@@ -113,6 +117,29 @@ def on_disconnected(disconnect_future):
     # Signal that sample is finished
     is_sample_done.set()
 
+# A list to hold all the pending jobs
+available_jobs = []
+def on_get_pending_job_executions_accepted(response):
+    # type: (iotjobs.GetPendingJobExecutionsResponse) -> None
+    with locked_data.lock:
+        if (len(response.queued_jobs) > 0 or len(response.in_progress_jobs) > 0):
+            print ("Pending Jobs:")
+            for job in response.in_progress_jobs:
+                available_jobs.append(job)
+                print(f"  In Progress: {job.job_id} @ {job.last_updated_at}")
+            for job in response.queued_jobs:
+                available_jobs.append(job)
+                print (f"  {job.job_id} @ {job.last_updated_at}")
+        else:
+            print ("No pending or queued jobs found!")
+        locked_data.got_job_response = True
+
+def on_get_pending_job_executions_rejected(error):
+    # type: (iotjobs.RejectedError) -> None
+    print (f"Request rejected: {error.code}: {error.message}")
+    exit("Get pending jobs request rejected!")
+
+
 def on_next_job_execution_changed(event):
     # type: (iotjobs.NextJobExecutionChangedEvent) -> None
     try:
@@ -214,8 +241,11 @@ def on_update_job_execution_rejected(rejected):
 
 if __name__ == '__main__':
     mqtt_connection = cmdUtils.build_mqtt_connection(None, None)
-    print("Connecting to {} with client ID '{}'...".format(
-        cmdUtils.get_command(cmdUtils.m_cmd_endpoint), cmdUtils.get_command("client_id")))
+    if is_ci == False:
+        print("Connecting to {} with client ID '{}'...".format(
+            cmdUtils.get_command(cmdUtils.m_cmd_endpoint), cmdUtils.get_command("client_id")))
+    else:
+        print("Connecting to endpoint with client ID")
 
     connected_future = mqtt_connection.connect()
 
@@ -229,6 +259,53 @@ def on_update_job_execution_rejected(rejected):
     connected_future.result()
     print("Connected!")
 
+    try:
+        # List the jobs queued and pending
+        get_jobs_request = iotjobs.GetPendingJobExecutionsRequest(thing_name=jobs_thing_name)
+        jobs_request_future_accepted, _ = jobs_client.subscribe_to_get_pending_job_executions_accepted(
+            request=get_jobs_request,
+            qos=mqtt.QoS.AT_LEAST_ONCE,
+            callback=on_get_pending_job_executions_accepted
+        )
+        # Wait for the subscription to succeed
+        jobs_request_future_accepted.result()
+
+        jobs_request_future_rejected, _ = jobs_client.subscribe_to_get_pending_job_executions_rejected(
+            request=get_jobs_request,
+            qos=mqtt.QoS.AT_LEAST_ONCE,
+            callback=on_get_pending_job_executions_rejected
+        )
+        # Wait for the subscription to succeed
+        jobs_request_future_rejected.result()
+
+        # Get a list of all the jobs
+        get_jobs_request_future = jobs_client.publish_get_pending_job_executions(
+            request=get_jobs_request,
+            qos=mqtt.QoS.AT_LEAST_ONCE
+        )
+        # Wait for the publish to succeed
+        get_jobs_request_future.result()
+    except Exception as e:
+        exit(e)
+
+    # If we are running in CI, then we want to check how many jobs were reported and stop
+    if (is_ci):
+        # Wait until we get a response. If we do not get a response after 50 tries, then abort
+        got_job_response_tries = 0
+        while (locked_data.got_job_response == False):
+            got_job_response_tries += 1
+            if (got_job_response_tries > 50):
+                exit("Got job response timeout exceeded")
+                sys.exit(-1)
+            time.sleep(0.2)
+
+        if (len(available_jobs) > 0):
+            print ("At least one job queued in CI! No further work to do. Exiting sample...")
+            sys.exit(0)
+        else:
+            print ("ERROR: No jobs queued in CI! At least one job should be queued!")
+            sys.exit(-1)
+
     try:
         # Subscribe to necessary topics.
         # Note that is **is** important to wait for "accepted/rejected" subscriptions
@@ -286,6 +363,7 @@ def on_update_job_execution_rejected(rejected):
         # Make initial attempt to start next job. The service should reply with
         # an "accepted" response, even if no jobs are pending. The response
         # will contain data about the next job, if there is one.
+        # (Will do nothing if we are in CI)
         try_start_next_job()
 
     except Exception as e:
diff --git a/samples/pkcs11_connect.py b/samples/pkcs11_connect.py
index 772a6b50..967df116 100644
--- a/samples/pkcs11_connect.py
+++ b/samples/pkcs11_connect.py
@@ -32,8 +32,10 @@
 cmdUtils.register_command("token_label", "", "Label of the PKCS#11 token to use (optional).")
 cmdUtils.register_command("slot_id", "", "Slot ID containing the PKCS#11 token to use (optional).", False, int)
 cmdUtils.register_command("key_label", "", "Label of private key on the PKCS#11 token (optional).")
+cmdUtils.register_command("is_ci", "", "If present the sample will run in CI mode (optional, default='None')")
 # Needs to be called so the command utils parse the commands
 cmdUtils.get_args()
+is_ci = cmdUtils.get_command("is_ci", None) != None
 
 # Callback when connection is accidentally lost.
 def on_connection_interrupted(connection, error, **kwargs):
@@ -50,8 +52,11 @@ def on_connection_resumed(connection, return_code, session_present, **kwargs):
     # (see build_pkcs11_mqtt_connection for implementation)
     mqtt_connection = cmdUtils.build_pkcs11_mqtt_connection(on_connection_interrupted, on_connection_resumed)
 
-    print("Connecting to {} with client ID '{}'...".format(
-        cmdUtils.get_command("endpoint"), cmdUtils.get_command("client_id")))
+    if is_ci == False:
+        print("Connecting to {} with client ID '{}'...".format(
+            cmdUtils.get_command(cmdUtils.m_cmd_endpoint), cmdUtils.get_command("client_id")))
+    else:
+        print("Connecting to endpoint with client ID")
 
     connect_future = mqtt_connection.connect()
 
diff --git a/samples/pubsub.py b/samples/pubsub.py
index 87bed58e..4977cb9d 100644
--- a/samples/pubsub.py
+++ b/samples/pubsub.py
@@ -26,11 +26,13 @@
 cmdUtils.register_command("port", "", "Connection port. AWS IoT supports 443 and 8883 (optional, default=auto).", type=int)
 cmdUtils.register_command("client_id", "", "Client ID to use for MQTT connection (optional, default='test-*').", default="test-" + str(uuid4()))
 cmdUtils.register_command("count", "", "The number of messages to send (optional, default='10').", default=10, type=int)
+cmdUtils.register_command("is_ci", "", "If present the sample will run in CI mode (optional, default='None')")
 # Needs to be called so the command utils parse the commands
 cmdUtils.get_args()
 
 received_count = 0
 received_all_event = threading.Event()
+is_ci = cmdUtils.get_command("is_ci", None) != None
 
 # Callback when connection is accidentally lost.
 def on_connection_interrupted(connection, error, **kwargs):
@@ -70,8 +72,11 @@ def on_message_received(topic, payload, dup, qos, retain, **kwargs):
 if __name__ == '__main__':
     mqtt_connection = cmdUtils.build_mqtt_connection(on_connection_interrupted, on_connection_resumed)
 
-    print("Connecting to {} with client ID '{}'...".format(
-        cmdUtils.get_command(cmdUtils.m_cmd_endpoint), cmdUtils.get_command("client_id")))
+    if is_ci == False:
+        print("Connecting to {} with client ID '{}'...".format(
+            cmdUtils.get_command(cmdUtils.m_cmd_endpoint), cmdUtils.get_command("client_id")))
+    else:
+        print("Connecting to endpoint with client ID")
     connect_future = mqtt_connection.connect()
 
     # Future.result() waits until a result is available
diff --git a/samples/shadow.py b/samples/shadow.py
index 7053cc25..6fc42377 100644
--- a/samples/shadow.py
+++ b/samples/shadow.py
@@ -1,6 +1,7 @@
 # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 # SPDX-License-Identifier: Apache-2.0.
 
+from time import sleep
 from awscrt import mqtt
 from awsiot import iotshadow
 from concurrent.futures import Future
@@ -40,6 +41,7 @@
 cmdUtils.register_command("client_id", "", "Client ID to use for MQTT connection (optional, default='test-*').", default="test-" + str(uuid4()))
 cmdUtils.register_command("thing_name", "", "The name assigned to your IoT Thing", required=True)
 cmdUtils.register_command("shadow_property", "", "The name of the shadow property you want to change (optional, default='color'", default="color")
+cmdUtils.register_command("is_ci", "", "If present the sample will run in CI mode (optional, default='None'. Will publish shadow automatically if set)")
 # Needs to be called so the command utils parse the commands
 cmdUtils.get_args()
 
@@ -48,6 +50,7 @@
 mqtt_connection = None
 shadow_thing_name = cmdUtils.get_command_required("thing_name")
 shadow_property = cmdUtils.get_command("shadow_property")
+is_ci = cmdUtils.get_command("is_ci", None) != None
 
 SHADOW_VALUE_DEFAULT = "off"
 
@@ -268,29 +271,47 @@ def change_shadow_value(value):
         future.add_done_callback(on_publish_update_shadow)
 
 def user_input_thread_fn():
-    while True:
-        try:
-            # Read user input
-            new_value = input()
+    # If we are not in CI, then take terminal input
+    if is_ci == False:
+        while True:
+            try:
+                # Read user input
+                new_value = input()
+
+                # If user wants to quit sample, then quit.
+                # Otherwise change the shadow value.
+                if new_value in ['exit', 'quit']:
+                    exit("User has quit")
+                    break
+                else:
+                    change_shadow_value(new_value)
 
-            # If user wants to quit sample, then quit.
-            # Otherwise change the shadow value.
-            if new_value in ['exit', 'quit']:
-                exit("User has quit")
+            except Exception as e:
+                print("Exception on input thread.")
+                exit(e)
                 break
-            else:
-                change_shadow_value(new_value)
-
+    # Otherwise, send shadow updates automatically
+    else:
+        try:
+            messages_sent = 0
+            while messages_sent < 5:
+                input = "Shadow_Value_" + str(messages_sent)
+                change_shadow_value(input)
+                sleep(1)
+                messages_sent += 1
+            exit("CI has quit")
         except Exception as e:
-            print("Exception on input thread.")
+            print ("Exception on input thread (CI)")
             exit(e)
-            break
 
 if __name__ == '__main__':
     mqtt_connection = cmdUtils.build_mqtt_connection(None, None)
 
-    print("Connecting to {} with client ID '{}'...".format(
-        cmdUtils.get_command(cmdUtils.m_cmd_endpoint), cmdUtils.get_command("client_id")))
+    if is_ci == False:
+        print("Connecting to {} with client ID '{}'...".format(
+            cmdUtils.get_command(cmdUtils.m_cmd_endpoint), cmdUtils.get_command("client_id")))
+    else:
+        print("Connecting to endpoint with client ID")
 
     connected_future = mqtt_connection.connect()
 
diff --git a/samples/websocket_connect.py b/samples/websocket_connect.py
index 2e6d3418..b90b5f03 100644
--- a/samples/websocket_connect.py
+++ b/samples/websocket_connect.py
@@ -18,8 +18,10 @@
 cmdUtils.register_command("client_id", "",
                           "Client ID to use for MQTT connection (optional, default='test-*').",
                           default="test-" + str(uuid4()))
+cmdUtils.register_command("is_ci", "", "If present the sample will run in CI mode (optional, default='None')")
 # Needs to be called so the command utils parse the commands
 cmdUtils.get_args()
+is_ci = cmdUtils.get_command("is_ci", None) != None
 
 # Callback when connection is accidentally lost.
 def on_connection_interrupted(connection, error, **kwargs):
@@ -36,8 +38,11 @@ def on_connection_resumed(connection, return_code, session_present, **kwargs):
     # (see build_websocket_mqtt_connection for implementation)
     mqtt_connection = cmdUtils.build_websocket_mqtt_connection(on_connection_interrupted, on_connection_resumed)
 
-    print("Connecting to {} with client ID '{}'...".format(
-        cmdUtils.get_command(cmdUtils.m_cmd_endpoint), cmdUtils.get_command("client_id")))
+    if is_ci == False:
+        print("Connecting to {} with client ID '{}'...".format(
+            cmdUtils.get_command(cmdUtils.m_cmd_endpoint), cmdUtils.get_command("client_id")))
+    else:
+        print ("Connecting to endpoint with client ID...")
 
     connect_future = mqtt_connection.connect()
 
diff --git a/samples/windows_cert_connect.py b/samples/windows_cert_connect.py
index 73800381..1f4b2497 100644
--- a/samples/windows_cert_connect.py
+++ b/samples/windows_cert_connect.py
@@ -20,9 +20,14 @@
 cmdUtils.register_command("client_id", "",
                           "Client ID to use for MQTT connection (optional, default='test-*').",
                           default="test-" + str(uuid4()))
+cmdUtils.register_command("is_ci", "", "If present the sample will run in CI mode (optional, default='None')")
+cmdUtils.register_command("cert", "", "Path to certificate in Windows cert store. "
+                            "e.g. \"CurrentUser\\MY\\6ac133ac58f0a88b83e9c794eba156a98da39b4c\"", True, str)
+cmdUtils.register_command("port", "", "Connection port. AWS IoT supports 443 and 8883 (optional, default=auto).", type=int)
 # Needs to be called so the command utils parse the commands
 cmdUtils.get_args()
 
+is_ci = cmdUtils.get_command("is_ci", None) != None
 
 def on_connection_interrupted(connection, error, **kwargs):
     # Callback when connection is accidentally lost.
@@ -47,8 +52,11 @@ def on_connection_resumed(connection, return_code, session_present, **kwargs):
         clean_session=False,
         keep_alive_secs=30)
 
-    print("Connecting to {} with client ID '{}'...".format(
-        cmdUtils.get_command(cmdUtils.m_cmd_endpoint), cmdUtils.get_command("client_id")))
+    if is_ci == False:
+        print("Connecting to {} with client ID '{}'...".format(
+            cmdUtils.get_command(cmdUtils.m_cmd_endpoint), cmdUtils.get_command("client_id")))
+    else:
+        print("Connecting to endpoint with client ID")
 
     connect_future = mqtt_connection.connect()
 
diff --git a/utils/delete_iot_thing_ci.py b/utils/delete_iot_thing_ci.py
new file mode 100644
index 00000000..b3d30a35
--- /dev/null
+++ b/utils/delete_iot_thing_ci.py
@@ -0,0 +1,67 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0.
+
+# Built-in
+import argparse
+import sys
+# Needs to be installed via pip
+import boto3  # - for launching sample
+
+def DeleteIoTThing(parsed_commands):
+    try:
+        iot_client = boto3.client('iot', region_name=parsed_commands.region)
+    except Exception:
+        print("Error - could not make Boto3 client. Credentials likely could not be sourced")
+        return -1
+
+    thing_principals = None
+    try:
+        thing_principals = iot_client.list_thing_principals(thingName=parsed_commands.thing_name)
+    except Exception:
+        print ("Could not get thing principals!")
+        return -1
+
+    try:
+        if (thing_principals != None):
+            if (thing_principals["principals"] != None):
+                if (len(thing_principals["principals"]) > 0 and parsed_commands.delete_certificate == "true"):
+                    for principal in thing_principals["principals"]:
+                        certificate_id = principal.split("/")[1]
+                        iot_client.detach_thing_principal(thingName=parsed_commands.thing_name, principal=principal)
+                        iot_client.update_certificate(certificateId=certificate_id, newStatus ='INACTIVE')
+                        iot_client.delete_certificate(certificateId=certificate_id, forceDelete=True)
+    except Exception as exception:
+        print (exception)
+        print ("Could not delete certificate!")
+        return -1
+
+    try:
+        iot_client.delete_thing(thingName=parsed_commands.thing_name)
+    except Exception as exception:
+        print (exception)
+        print ("Could not delete IoT thing!")
+        return -1
+
+    print ("IoT thing deleted successfully")
+    return 0
+
+
+
+def main():
+    argument_parser = argparse.ArgumentParser(
+        description="Delete IoT Thing")
+    argument_parser.add_argument("--thing_name", metavar="", required=True,
+                                 help="The name of the IoT thing to delete")
+    argument_parser.add_argument("--region", metavar="",
+                                 required=True, default="us-east-1", help="The name of the region to use")
+    argument_parser.add_argument("--delete_certificate", metavar="",
+                                 required=False, default="true", help="Will delete the certificate after detaching it from the IoT thing")
+    parsed_commands = argument_parser.parse_args()
+
+    print ("Deleting IoT thing...")
+    delete_result = DeleteIoTThing(parsed_commands)
+    sys.exit(delete_result)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/utils/run_sample_ci.py b/utils/run_sample_ci.py
new file mode 100644
index 00000000..34d38929
--- /dev/null
+++ b/utils/run_sample_ci.py
@@ -0,0 +1,348 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0.
+
+# Built-in
+import argparse
+import os
+import subprocess
+import pathlib
+import sys
+# Needs to be installed via pip
+import boto3  # - for launching sample
+
+current_folder = os.path.dirname(pathlib.Path(__file__).resolve())
+if sys.platform == "win32" or sys.platform == "cygwin":
+    current_folder += "\\"
+else:
+    current_folder += "/"
+tmp_certificate_file_path = str(current_folder) + "tmp_certificate.pem"
+tmp_private_key_path = str(current_folder) + "tmp_privatekey.pem.key"
+tmp_pfx_file_path = str(current_folder) + "tmp_pfx_certificate.pfx"
+tmp_pfx_certificate_path = ""
+tmp_pfx_certificate_store_location = "CurrentUser\\My"
+tmp_pfx_password = "" # Setting a password causes issues, but an empty string is valid so we use that
+
+
+def get_secrets_and_launch(parsed_commands):
+    global tmp_certificate_file_path
+    global tmp_private_key_path
+    global tmp_pfx_file_path
+    global tmp_pfx_certificate_path
+    exit_code = 0
+    sample_endpoint = ""
+    sample_certificate = ""
+    sample_private_key = ""
+    sample_custom_authorizer_name = ""
+    sample_custom_authorizer_password = ""
+
+    print("Attempting to get credentials from secrets using Boto3...")
+    secrets_client = boto3.client(
+        "secretsmanager", region_name=parsed_commands.sample_region)
+    try:
+        if (parsed_commands.sample_secret_endpoint != ""):
+            sample_endpoint = secrets_client.get_secret_value(
+                SecretId=parsed_commands.sample_secret_endpoint)["SecretString"]
+        if (parsed_commands.sample_secret_certificate != ""):
+            secret_data = secrets_client.get_secret_value(
+                SecretId=parsed_commands.sample_secret_certificate)
+            with open(tmp_certificate_file_path, "w") as file:
+                # lgtm [py/clear-text-storage-sensitive-data]
+                file.write(secret_data["SecretString"])
+            sample_certificate = tmp_certificate_file_path
+        if (parsed_commands.sample_secret_private_key != ""):
+            secret_data = secrets_client.get_secret_value(
+                SecretId=parsed_commands.sample_secret_private_key)
+            with open(tmp_private_key_path, "w") as file:
+                # lgtm [py/clear-text-storage-sensitive-data]
+                file.write(secret_data["SecretString"])
+            sample_private_key = tmp_private_key_path
+        if (parsed_commands.sample_secret_custom_authorizer_name != ""):
+            sample_custom_authorizer_name = secrets_client.get_secret_value(
+                SecretId=parsed_commands.sample_secret_custom_authorizer_name)["SecretString"]
+        if (parsed_commands.sample_secret_custom_authorizer_password != ""):
+            sample_custom_authorizer_password = secrets_client.get_secret_value(
+                SecretId=parsed_commands.sample_secret_custom_authorizer_password)["SecretString"]
+
+    except Exception:
+        sys.exit("ERROR: Could not get secrets to launch sample!")
+
+    extra_step_return = 0
+    if (parsed_commands.sample_run_softhsm != ""):
+        extra_step_return = make_softhsm_key()
+        sample_private_key = "" # Do not use the private key
+    if (parsed_commands.sample_run_certutil != ""):
+        extra_step_return = make_windows_pfx_file()
+        sample_private_key = "" # Do not use the private key
+        sample_certificate = tmp_pfx_certificate_path # use the Windows certificate path
+
+    exit_code = extra_step_return
+    if (extra_step_return == 0):
+        print("Launching sample...")
+        exit_code = launch_sample(parsed_commands, sample_endpoint, sample_certificate,
+                                sample_private_key, sample_custom_authorizer_name, sample_custom_authorizer_password)
+
+        if (exit_code == 0):
+            print("SUCCESS: Finished running sample! Exiting with success")
+        else:
+            print("ERROR: Sample did not return success! Exit code " + str(exit_code))
+    else:
+        print ("ERROR: Could not run extra step (SoftHSM, CertUtil, etc)")
+
+    print("Deleting files...")
+    if (os.path.isfile(tmp_certificate_file_path)):
+        os.remove(tmp_certificate_file_path)
+    if (os.path.isfile(tmp_private_key_path)):
+        os.remove(tmp_private_key_path)
+    if (os.path.isfile(tmp_pfx_file_path)):
+        os.remove(tmp_pfx_file_path)
+
+    return exit_code
+
+
+def make_softhsm_key():
+    print ("Setting up private key via SoftHSM")
+    softhsm_run = subprocess.run("softhsm2-util --init-token --free --label my-token --pin 0000 --so-pin 0000", shell=True)
+    if (softhsm_run.returncode != 0):
+        print ("ERROR: SoftHSM could not initialize a new token")
+        return softhsm_run.returncode
+    softhsm_run = subprocess.run(f"softhsm2-util --import {tmp_private_key_path} --token my-token --label my-key --id BEEFCAFE --pin 0000", shell=True)
+    if (softhsm_run.returncode != 0):
+        print ("ERROR: SoftHSM could not import token")
+    print ("Finished setting up private key in SoftHSM")
+    return 0
+
+
+def make_windows_pfx_file():
+    global tmp_certificate_file_path
+    global tmp_private_key_path
+    global tmp_pfx_file_path
+    global tmp_pfx_certificate_path
+
+    if sys.platform == "win32" or sys.platform == "cygwin":
+        if os.path.isfile(tmp_certificate_file_path) != True:
+            print (tmp_certificate_file_path)
+            print("ERROR: Certificate file not found!")
+            return 1
+        if os.path.isfile(tmp_private_key_path) != True:
+            print("ERROR: Private key file not found!")
+            return 1
+
+        # Delete old PFX file if it exists
+        if os.path.isfile(tmp_pfx_file_path):
+            os.remove(tmp_pfx_file_path)
+
+        # Make a key copy
+        copy_path = os.path.splitext(tmp_certificate_file_path)
+        with open(copy_path[0] + ".key", 'w') as file:
+            key_file = open(tmp_private_key_path)
+            file.write(key_file.read())
+            key_file.close()
+
+        # Make a PFX file
+        certutil_error_occurred = False
+        arguments = ["certutil",  "-mergePFX", tmp_certificate_file_path, tmp_pfx_file_path]
+        certutil_run = subprocess.run(args=arguments, shell=True, input=f"{tmp_pfx_password}\n{tmp_pfx_password}", encoding='ascii')
+        if (certutil_run.returncode != 0):
+            print ("ERROR: Could not make PFX file")
+            certutil_error_occurred = True
+            return 1
+        else:
+            print ("PFX file created successfully")
+
+        # Remove the temporary key copy
+        if os.path.isfile(copy_path[0] + ".key"):
+            os.remove(copy_path[0] + ".key")
+        if (certutil_error_occurred == True):
+            return 1
+
+        # Import the PFX into the Windows Certificate Store
+        # (Passing '$mypwd' is required even though it is empty and our certificate has no password. It fails CI otherwise)
+        import_pfx_arguments = ["powershell.exe", "Import-PfxCertificate", "-FilePath", tmp_pfx_file_path, "-CertStoreLocation", "Cert:\\" + tmp_pfx_certificate_store_location, "-Password", "$mypwd"]
+        import_pfx_run = subprocess.run(args=import_pfx_arguments, shell=True, stdout=subprocess.PIPE)
+        if (import_pfx_run.returncode != 0):
+            print ("ERROR: Could not import PFX certificate into Windows store!")
+            return 1
+        else:
+            print ("Certificate imported to Windows Certificate Store successfully")
+
+        # Get the certificate thumbprint from the output:
+        import_pfx_output = str(import_pfx_run.stdout)
+        # We know the Thumbprint will always be 40 characters long, so we can find it using that
+        # TODO: Extract this using a better method
+        thumbprint = ""
+        current_str = ""
+        # The input comes as a string with some special characters still included, so we need to remove them!
+        import_pfx_output = import_pfx_output.replace("\\r", " ")
+        import_pfx_output = import_pfx_output.replace("\\n", "\n")
+        for i in range(0, len(import_pfx_output)):
+            if (import_pfx_output[i] == " " or import_pfx_output[i] == "\n"):
+                if (len(current_str) == 40):
+                    thumbprint = current_str
+                    break
+                current_str = ""
+            else:
+                current_str += import_pfx_output[i]
+
+        # Did we get a thumbprint?
+        if (thumbprint == ""):
+            print ("ERROR: Could not find certificate thumbprint")
+            return 1
+
+        # Construct the certificate path
+        tmp_pfx_certificate_path = tmp_pfx_certificate_store_location + "\\" + thumbprint
+
+        # Return success
+        print ("PFX certificate created and imported successfully!")
+        return 0
+
+    else:
+        print("ERROR - Windows PFX file can only be created on a Windows platform!")
+        return 1
+
+
+def launch_sample(parsed_commands, sample_endpoint, sample_certificate, sample_private_key, sample_custom_authorizer_name, sample_custom_authorizer_password):
+    global tmp_certificate_file_path
+    global tmp_private_key_path
+    global tmp_pfx_file_path
+    exit_code = 0
+
+    print("Processing arguments...")
+    launch_arguments = []
+    launch_arguments.append("--endpoint")
+    launch_arguments.append(sample_endpoint)
+
+    if (sample_certificate != ""):
+        launch_arguments.append("--cert")
+        launch_arguments.append(sample_certificate)
+    if (sample_private_key != ""):
+        launch_arguments.append("--key")
+        launch_arguments.append(sample_private_key)
+    if (sample_custom_authorizer_name != ""):
+        launch_arguments.append("--custom_auth_authorizer_name")
+        launch_arguments.append(sample_custom_authorizer_name)
+    if (sample_custom_authorizer_password != ""):
+        launch_arguments.append("--custom_auth_password")
+        launch_arguments.append(sample_custom_authorizer_password)
+    if (parsed_commands.sample_arguments != ""):
+        sample_arguments_split = parsed_commands.sample_arguments.split(" ")
+        for arg in sample_arguments_split:
+            launch_arguments.append(arg)
+
+    print("Launching sample...")
+    # Based on the programming language, we have to run it a different way
+    if (parsed_commands.language == "Java"):
+        arguments_as_string = ""
+        for i in range(0, len(launch_arguments)):
+            arguments_as_string += str(launch_arguments[i])
+            if (i+1 < len(launch_arguments)):
+                arguments_as_string += " "
+        arguments = ["mvn", "compile", "exec:java"]
+        arguments.append("-pl")
+        arguments.append(parsed_commands.sample_file)
+        arguments.append("-Dexec.mainClass=" +
+                         parsed_commands.sample_main_class)
+        arguments.append("-Daws.crt.ci=True")
+
+        # We have to do this as a string, unfortunately, due to how -Dexec.args= works...
+        argument_string = subprocess.list2cmdline(
+            arguments) + " -Dexec.args=\"" + arguments_as_string + "\""
+        sample_return = subprocess.run(argument_string, shell=True)
+        exit_code = sample_return.returncode
+
+    elif (parsed_commands.language == "CPP"):
+        sample_return = subprocess.run(
+            args=launch_arguments, executable=parsed_commands.sample_file)
+        exit_code = sample_return.returncode
+
+    elif (parsed_commands.language == "Python"):
+        launch_arguments.append("--is_ci")
+        launch_arguments.append("True")
+
+        sample_return = subprocess.run(
+            args=[sys.executable, parsed_commands.sample_file] + launch_arguments)
+        exit_code = sample_return.returncode
+
+    elif (parsed_commands.language == "Javascript"):
+        os.chdir(parsed_commands.sample_file)
+
+        launch_arguments.append("--is_ci")
+        launch_arguments.append("true")
+
+        sample_return_one = None
+        if sys.platform == "win32" or sys.platform == "cygwin":
+            sample_return_one = subprocess.run(args=["npm", "install"], shell=True)
+        else:
+            sample_return_one = subprocess.run(args=["npm", "install"])
+
+        if (sample_return_one == None or sample_return_one.returncode != 0):
+            exit_code = sample_return_one.returncode
+        else:
+            sample_return_two = None
+            arguments = []
+            if (parsed_commands.node_cmd == "" or parsed_commands.node_cmd == None):
+                arguments = ["node", "dist/index.js"]
+            else:
+                arguments = parsed_commands.node_cmd.split(" ")
+
+            if sys.platform == "win32" or sys.platform == "cygwin":
+                sample_return_two = subprocess.run(
+                    args=arguments + launch_arguments, shell=True)
+            else:
+                sample_return_two = subprocess.run(
+                    args=arguments + launch_arguments)
+
+            if (sample_return_two != None):
+                exit_code = sample_return_two.returncode
+            else:
+                exit_code = 1
+
+    else:
+        print("ERROR - unknown programming language! Supported programming languages are 'Java', 'CPP', 'Python', and 'Javascript'")
+        return -1
+
+    # finish!
+    return exit_code
+
+
+def main():
+    argument_parser = argparse.ArgumentParser(
+        description="Run Sample in CI")
+    argument_parser.add_argument("--language", metavar="", required=True,
+                                 help="The name of the programming language. Used to determine how to launch the sample")
+    argument_parser.add_argument("--sample_file",
+                                 metavar="",
+                                 required=True, default="", help="Sample to launch. Format varies based on programming language")
+    argument_parser.add_argument("--sample_region", metavar="",
+                                 required=True, default="us-east-1", help="The name of the region to use for accessing secrets")
+    argument_parser.add_argument("--sample_secret_endpoint", metavar="",
+                                 required=False, default="", help="The name of the secret containing the endpoint")
+    argument_parser.add_argument("--sample_secret_certificate", metavar="", required=False,
+                                 default="", help="The name of the secret containing the certificate PEM file")
+    argument_parser.add_argument("--sample_secret_private_key", metavar="", required=False,
+                                 default="", help="The name of the secret containing the private key PEM file")
+    argument_parser.add_argument("--sample_secret_custom_authorizer_name", metavar="", required=False,
+                                 default="", help="The name of the secret containing the custom authorizer name")
+    argument_parser.add_argument("--sample_secret_custom_authorizer_password", metavar="", required=False,
+                                 default="", help="The name of the secret containing the custom authorizer password")
+    argument_parser.add_argument("--sample_run_softhsm", metavar="", required=False,
+                                 default="", help="Runs SoftHSM on the private key passed, storing it, rather than passing it directly to the sample. Used for PKCS11 sample")
+    argument_parser.add_argument("--sample_run_certutil", metavar="", required=False,
+                                 default="", help="Runs CertUtil on the private key and certificate passed and makes a certificate.pfx file, "
+                                 "which is used automatically in the --cert argument. Used for Windows Certificate Connect sample")
+    argument_parser.add_argument("--sample_arguments", metavar="",
+                                 required=False, default="",
+                                 help="Arguments to pass to sample. In Java, these arguments will be in a double quote (\") string")
+    argument_parser.add_argument("--sample_main_class", metavar="",
+                                 required=False, default="", help="Java only: The main class to run")
+    argument_parser.add_argument("--node_cmd", metavar="", required=False, default="",
+                                 help="Javascript only: Overrides the default 'npm dist/index.js' with whatever you pass. Useful for launching pure Javascript samples")
+
+    parsed_commands = argument_parser.parse_args()
+
+    print("Starting to launch sample...")
+    sample_result = get_secrets_and_launch(parsed_commands)
+    sys.exit(sample_result)
+
+
+if __name__ == "__main__":
+    main()