From 463dd62ccf3304c09463cd5fb548ab69ea51fe0d Mon Sep 17 00:00:00 2001 From: Vera Xia Date: Wed, 1 Jun 2022 20:14:05 -0700 Subject: [PATCH 1/2] update DA script, Delete Test Resources for Device Advisor --- builder.json | 3 ++- deviceadvisor/script/DATestRun.py | 28 ++++++++++++++++++++++++++-- 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/builder.json b/builder.json index b00072dd..2ce80e82 100644 --- a/builder.json +++ b/builder.json @@ -96,6 +96,7 @@ "DA_TOPIC": "test/da", "DA_SHADOW_PROPERTY": "datest", "DA_SHADOW_VALUE_SET": "ON", - "DA_SHADOW_VALUE_DEFAULT": "OFF" + "DA_SHADOW_VALUE_DEFAULT": "OFF", + "DA_S3_NAME": "aws-iot-sdk-deviceadvisor-logs" } } diff --git a/deviceadvisor/script/DATestRun.py b/deviceadvisor/script/DATestRun.py index 20d40922..ae454f2e 100644 --- a/deviceadvisor/script/DATestRun.py +++ b/deviceadvisor/script/DATestRun.py @@ -3,6 +3,7 @@ import json import os import subprocess +import re from time import sleep ############################################## @@ -19,6 +20,20 @@ def delete_thing_with_certi(thingName, certiId, certiArn): os.remove(os.environ["DA_CERTI"]) os.remove(os.environ["DA_KEY"]) +# Export the testing log and upload it to S3 bucket +def process_logs(log_group, log_stream, thing_name): + logs_client = boto3.client('logs') + response = logs_client.get_log_events( + logGroupName=log_group, + logStreamName=log_stream + ) + log_file = thing_name + ".log" + f = open(log_file, 'w') + for event in response["events"]: + f.write(event['message']) + f.close() + s3.Bucket(os.environ['DA_S3_NAME']).upload_file(log_file, log_file) + os.remove(log_file) ############################################## # Initialize variables @@ -26,6 +41,7 @@ def delete_thing_with_certi(thingName, certiId, certiArn): client = boto3.client('iot') dataClient = boto3.client('iot-data') deviceAdvisor = boto3.client('iotdeviceadvisor') +s3 = boto3.resource('s3') # load test config f = open('deviceadvisor/script/DATestConfig.json') @@ -192,10 +208,18 @@ def delete_thing_with_certi(thingName, certiId, certiArn): # If the test finalizing then store the test result elif (test_result_responds['status'] != 'RUNNING'): test_result[test_name] = test_result_responds['status'] - if(test_result[test_name] == "PASS"): - delete_thing_with_certi(thing_name, certificate_id ,certificate_arn ) + # If the test failed, upload the logs to S3 before clean up + if(test_result[test_name] != "PASS"): + log_url = test_result_responds['testResult']['groups'][0]['tests'][0]['logUrl'] + group_string = re.search('group=(.*);', log_url) + log_group = group_string.group(1) + stream_string = re.search('stream=(.*)', log_url) + log_stream = stream_string.group(1) + process_logs(log_group, log_stream, thing_name) + delete_thing_with_certi(thing_name, certificate_id ,certificate_arn) break except Exception as e: + delete_thing_with_certi(thing_name, certificate_id ,certificate_arn) print("[Device Advisor]Error: Failed to test: "+ test_name + e) exit(-1) From 34ad1f5583ba67d01a7b27b3f3d7e33022b9b5a9 Mon Sep 17 00:00:00 2001 From: Vera Xia Date: Thu, 2 Jun 2022 14:09:49 -0700 Subject: [PATCH 2/2] add backoff for device advisor requests --- deviceadvisor/script/DATestRun.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/deviceadvisor/script/DATestRun.py b/deviceadvisor/script/DATestRun.py index ae454f2e..782630bf 100644 --- a/deviceadvisor/script/DATestRun.py +++ b/deviceadvisor/script/DATestRun.py @@ -4,6 +4,7 @@ import os import subprocess import re +import random from time import sleep ############################################## @@ -34,6 +35,11 @@ def process_logs(log_group, log_stream, thing_name): f.close() s3.Bucket(os.environ['DA_S3_NAME']).upload_file(log_file, log_file) os.remove(log_file) + print("[Device Advisor] Device Advisor Log file uploaded to "+ log_file) + +# Sleep for a random time between base and bax +def sleep_with_backoff(base, max): + sleep(random.randint(base,max)) ############################################## # Initialize variables @@ -43,6 +49,10 @@ def process_logs(log_group, log_stream, thing_name): deviceAdvisor = boto3.client('iotdeviceadvisor') s3 = boto3.resource('s3') +# const +BACKOFF_BASE = 5 +BACKOFF_MAX = 10 + # load test config f = open('deviceadvisor/script/DATestConfig.json') DATestConfig = json.load(f) @@ -171,6 +181,7 @@ def process_logs(log_group, log_stream, thing_name): # 'createdAt': datetime(2015, 1, 1) # } print("[Device Advisor]Info: Start device advisor test: " + test_name) + sleep_with_backoff(BACKOFF_BASE, BACKOFF_MAX) test_start_response = deviceAdvisor.start_suite_run( suiteDefinitionId=DATestConfig['test_suite_ids'][test_name], suiteRunConfiguration={ @@ -187,8 +198,8 @@ def process_logs(log_group, log_stream, thing_name): os.environ['DA_ENDPOINT'] = endpoint_response['endpoint'] while True: - # sleep for 1s every loop to avoid TooManyRequestsException - sleep(1) + # Add backoff to avoid TooManyRequestsException + sleep_with_backoff(BACKOFF_BASE, BACKOFF_MAX) test_result_responds = deviceAdvisor.get_suite_run( suiteDefinitionId=DATestConfig['test_suite_ids'][test_name], suiteRunId=test_start_response['suiteRunId']