diff --git a/builder.json b/builder.json index b00072dd..2ce80e82 100644 --- a/builder.json +++ b/builder.json @@ -96,6 +96,7 @@ "DA_TOPIC": "test/da", "DA_SHADOW_PROPERTY": "datest", "DA_SHADOW_VALUE_SET": "ON", - "DA_SHADOW_VALUE_DEFAULT": "OFF" + "DA_SHADOW_VALUE_DEFAULT": "OFF", + "DA_S3_NAME": "aws-iot-sdk-deviceadvisor-logs" } } diff --git a/deviceadvisor/script/DATestRun.py b/deviceadvisor/script/DATestRun.py index 20d40922..782630bf 100644 --- a/deviceadvisor/script/DATestRun.py +++ b/deviceadvisor/script/DATestRun.py @@ -3,6 +3,8 @@ import json import os import subprocess +import re +import random from time import sleep ############################################## @@ -19,6 +21,25 @@ def delete_thing_with_certi(thingName, certiId, certiArn): os.remove(os.environ["DA_CERTI"]) os.remove(os.environ["DA_KEY"]) +# Export the testing log and upload it to S3 bucket +def process_logs(log_group, log_stream, thing_name): + logs_client = boto3.client('logs') + response = logs_client.get_log_events( + logGroupName=log_group, + logStreamName=log_stream + ) + log_file = thing_name + ".log" + f = open(log_file, 'w') + for event in response["events"]: + f.write(event['message']) + f.close() + s3.Bucket(os.environ['DA_S3_NAME']).upload_file(log_file, log_file) + os.remove(log_file) + print("[Device Advisor] Device Advisor Log file uploaded to "+ log_file) + +# Sleep for a random time between base and bax +def sleep_with_backoff(base, max): + sleep(random.randint(base,max)) ############################################## # Initialize variables @@ -26,6 +47,11 @@ def delete_thing_with_certi(thingName, certiId, certiArn): client = boto3.client('iot') dataClient = boto3.client('iot-data') deviceAdvisor = boto3.client('iotdeviceadvisor') +s3 = boto3.resource('s3') + +# const +BACKOFF_BASE = 5 +BACKOFF_MAX = 10 # load test config f = open('deviceadvisor/script/DATestConfig.json') @@ -155,6 +181,7 @@ def delete_thing_with_certi(thingName, certiId, certiArn): # 'createdAt': datetime(2015, 1, 1) # } print("[Device Advisor]Info: Start device advisor test: " + test_name) + sleep_with_backoff(BACKOFF_BASE, BACKOFF_MAX) test_start_response = deviceAdvisor.start_suite_run( suiteDefinitionId=DATestConfig['test_suite_ids'][test_name], suiteRunConfiguration={ @@ -171,8 +198,8 @@ def delete_thing_with_certi(thingName, certiId, certiArn): os.environ['DA_ENDPOINT'] = endpoint_response['endpoint'] while True: - # sleep for 1s every loop to avoid TooManyRequestsException - sleep(1) + # Add backoff to avoid TooManyRequestsException + sleep_with_backoff(BACKOFF_BASE, BACKOFF_MAX) test_result_responds = deviceAdvisor.get_suite_run( suiteDefinitionId=DATestConfig['test_suite_ids'][test_name], suiteRunId=test_start_response['suiteRunId'] @@ -192,10 +219,18 @@ def delete_thing_with_certi(thingName, certiId, certiArn): # If the test finalizing then store the test result elif (test_result_responds['status'] != 'RUNNING'): test_result[test_name] = test_result_responds['status'] - if(test_result[test_name] == "PASS"): - delete_thing_with_certi(thing_name, certificate_id ,certificate_arn ) + # If the test failed, upload the logs to S3 before clean up + if(test_result[test_name] != "PASS"): + log_url = test_result_responds['testResult']['groups'][0]['tests'][0]['logUrl'] + group_string = re.search('group=(.*);', log_url) + log_group = group_string.group(1) + stream_string = re.search('stream=(.*)', log_url) + log_stream = stream_string.group(1) + process_logs(log_group, log_stream, thing_name) + delete_thing_with_certi(thing_name, certificate_id ,certificate_arn) break except Exception as e: + delete_thing_with_certi(thing_name, certificate_id ,certificate_arn) print("[Device Advisor]Error: Failed to test: "+ test_name + e) exit(-1)