Shadow Update Service Test (#539)

alfred2g · web-flow · commit e297944193b8 · 2023-12-20T15:25:43.000-08:00
* Shadow Update Service Test

* Add command input

* Change shadow file name

* Fix executable path

* Add mqtt version number

* Fix named shadow path

* Add shadow name parameter

* Add Named Shadow support

* Debug: don't delete thing

* create named shadow from script

* create shadow

* Add mqtt5 support

* removes unsused code

* Remove dead code

* Fix comments
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -34,6 +34,7 @@ env:
   CI_BUILD_AND_TEST_ROLE: arn:aws:iam::180635532705:role/V2_SDK_Unit_Testing
   CI_JOBS_SERVICE_CLIENT_ROLE: arn:aws:iam::180635532705:role/CI_JobsServiceClient_Role
   CI_SERVICE_ROLE_CFG_FOLDER: "./aws-iot-device-sdk-python-v2/servicetests/test_cases"
+  CI_SHADOW_SERVICE_CLIENT_ROLE: arn:aws:iam::180635532705:role/CI_ShadowServiceClient_Role
 
 jobs:
 
@@ -207,6 +208,32 @@ jobs:
           chmod a+x builder
           ./builder build -p ${{ env.PACKAGE_NAME }}
 
+      - name: configure AWS credentials (service tests Shadow)
+        uses: aws-actions/configure-aws-credentials@v2
+        with:
+          role-to-assume: ${{ env.CI_SHADOW_SERVICE_CLIENT_ROLE }}
+          aws-region: ${{ env.AWS_DEFAULT_REGION }}
+      - name: run MQTT5 Shadow Update
+        working-directory: ./aws-iot-device-sdk-python-v2/servicetests
+        run: |
+          export PYTHONPATH=${{ github.workspace }}/aws-iot-device-sdk-python-v2/utils:${{ github.workspace }}/aws-iot-device-sdk-python-v2/samples
+          python3 ./test_cases/test_shadow_update.py --config-file test_cases/mqtt5_shadow_cfg.json
+      - name: run MQTT3 Shadow Update
+        working-directory: ./aws-iot-device-sdk-python-v2/servicetests
+        run: |
+          export PYTHONPATH=${{ github.workspace }}/aws-iot-device-sdk-python-v2/utils:${{ github.workspace }}/aws-iot-device-sdk-python-v2/samples
+          python3 ./test_cases/test_shadow_update.py --config-file test_cases/mqtt3_shadow_cfg.json
+      - name: run MQTT5 Named Shadow Update
+        working-directory: ./aws-iot-device-sdk-python-v2/servicetests
+        run: |
+          export PYTHONPATH=${{ github.workspace }}/aws-iot-device-sdk-python-v2/utils:${{ github.workspace }}/aws-iot-device-sdk-python-v2/samples
+          python3 ./test_cases/test_shadow_update.py --config-file test_cases/mqtt5_named_shadow_cfg.json
+      - name: run MQTT3 Named Shadow Update
+        working-directory: ./aws-iot-device-sdk-python-v2/servicetests
+        run: |
+          export PYTHONPATH=${{ github.workspace }}/aws-iot-device-sdk-python-v2/utils:${{ github.workspace }}/aws-iot-device-sdk-python-v2/samples
+          python3 ./test_cases/test_shadow_update.py --config-file test_cases/mqtt3_named_shadow_cfg.json
+
       - name: configure AWS credentials (service tests Jobs)
         uses: aws-actions/configure-aws-credentials@v2
         with:
diff --git a/samples/utils/command_line_utils.py b/samples/utils/command_line_utils.py
@@ -289,6 +289,8 @@ class CmdData:
         input_job_time : int
         # Shadow
         input_shadow_property : str
+        input_shadow_value : str
+        input_shadow_name : str
         # PKCS12
         input_pkcs12_file : str
         input_pkcs12_password : str
@@ -703,7 +705,10 @@ def parse_sample_input_shadow():
         cmdUtils.register_command(CommandLineUtils.m_cmd_port, "<int>", "Connection port. AWS IoT supports 443 and 8883 (optional, default=8883).", type=int)
         cmdUtils.register_command(CommandLineUtils.m_cmd_client_id, "<str>", "Client ID to use for MQTT connection (optional, default='test-*').", default="test-" + str(uuid4()))
         cmdUtils.register_command(CommandLineUtils.m_cmd_thing_name, "<str>", "The name assigned to your IoT Thing", required=True)
-        cmdUtils.register_command(CommandLineUtils.m_cmd_shadow_property, "<str>", "The name of the shadow property you want to change (optional, default='color'", default="color")
+        cmdUtils.register_command(CommandLineUtils.m_cmd_shadow_property, "<str>", "The name of the shadow property you want to change (optional, default=''", default="")
+        cmdUtils.register_command(CommandLineUtils.m_cmd_shadow_value, "<str>", "The desired value of the shadow property you want to set (optional)")
+        cmdUtils.register_command(CommandLineUtils.m_cmd_shadow_name, "<str>", "Shadow name (optional, default='')", type=str)
+        cmdUtils.register_command(CommandLineUtils.m_cmd_mqtt_version, "<int>", "mqtt version (optional, default='5')", default=5, type=int)
         cmdUtils.get_args()
 
         cmdData = CommandLineUtils.CmdData()
@@ -717,7 +722,10 @@ def parse_sample_input_shadow():
         cmdData.input_proxy_port = int(cmdUtils.get_command(CommandLineUtils.m_cmd_proxy_port))
         cmdData.input_thing_name = cmdUtils.get_command_required(CommandLineUtils.m_cmd_thing_name)
         cmdData.input_shadow_property = cmdUtils.get_command_required(CommandLineUtils.m_cmd_shadow_property)
+        cmdData.input_shadow_value = cmdUtils.get_command(CommandLineUtils.m_cmd_shadow_value, None)
+        cmdData.input_shadow_name = cmdUtils.get_command(CommandLineUtils.m_cmd_shadow_name, None)
         cmdData.input_is_ci = cmdUtils.get_command(CommandLineUtils.m_cmd_is_ci, None) != None
+        cmdData.input_mqtt_version = int(cmdUtils.get_command(CommandLineUtils.m_cmd_mqtt_version, 5))
         return cmdData
 
     def parse_sample_input_websocket_connect():
@@ -876,6 +884,8 @@ def parse_sample_input_pkcs12_connect():
     m_cmd_count = "count"
     m_cmd_group_identifier = "group_identifier"
     m_cmd_shadow_property = "shadow_property"
+    m_cmd_shadow_value = "shadow_value"
+    m_cmd_shadow_name = "shadow_name"
     m_cmd_pkcs12_file = "pkcs12_file"
     m_cmd_pkcs12_password = "pkcs12_password"
     m_cmd_region = "region"
diff --git a/servicetests/test_cases/mqtt3_named_shadow_cfg.json b/servicetests/test_cases/mqtt3_named_shadow_cfg.json
@@ -0,0 +1,40 @@
+{
+    "language": "Python",
+    "runnable_file": "./tests/ShadowUpdate/shadow_update.py",
+    "runnable_region": "us-east-1",
+    "runnable_main_class": "",
+    "arguments": [
+        {
+            "name": "--mqtt_version",
+            "data": "3"
+        },
+        {
+            "name": "--endpoint",
+            "secret": "ci/endpoint"
+        },
+        {
+            "name": "--cert",
+            "data": "tests/ShadowUpdate/certificate.pem.crt"
+        },
+        {
+            "name": "--key",
+            "data": "tests/ShadowUpdate/private.pem.key"
+        },
+        {
+            "name": "--thing_name",
+            "data": "ServiceTest_Shadow_$INPUT_UUID"
+        },
+        {
+            "name": "--shadow_property",
+            "data": "color"
+        },
+        {
+            "name": "--shadow_value",
+            "data": "on"
+        },
+        {
+            "name": "--shadow_name",
+            "data": "testShadow"
+        }
+    ]
+}
diff --git a/servicetests/test_cases/mqtt3_shadow_cfg.json b/servicetests/test_cases/mqtt3_shadow_cfg.json
@@ -0,0 +1,36 @@
+{
+    "language": "Python",
+    "runnable_file": "./tests/ShadowUpdate/shadow_update.py",
+    "runnable_region": "us-east-1",
+    "runnable_main_class": "",
+    "arguments": [
+        {
+            "name": "--mqtt_version",
+            "data": "3"
+        },
+        {
+            "name": "--endpoint",
+            "secret": "ci/endpoint"
+        },
+        {
+            "name": "--cert",
+            "data": "tests/ShadowUpdate/certificate.pem.crt"
+        },
+        {
+            "name": "--key",
+            "data": "tests/ShadowUpdate/private.pem.key"
+        },
+        {
+            "name": "--thing_name",
+            "data": "ServiceTest_Shadow_$INPUT_UUID"
+        },
+        {
+            "name": "--shadow_property",
+            "data": "color"
+        },
+        {
+            "name": "--shadow_value",
+            "data": "on"
+        }
+    ]
+}
diff --git a/servicetests/test_cases/mqtt5_named_shadow_cfg.json b/servicetests/test_cases/mqtt5_named_shadow_cfg.json
@@ -0,0 +1,40 @@
+{
+    "language": "Python",
+    "runnable_file": "./tests/ShadowUpdate/shadow_update.py",
+    "runnable_region": "us-east-1",
+    "runnable_main_class": "",
+    "arguments": [
+        {
+            "name": "--mqtt_version",
+            "data": "5"
+        },
+        {
+            "name": "--endpoint",
+            "secret": "ci/endpoint"
+        },
+        {
+            "name": "--cert",
+            "data": "tests/ShadowUpdate/certificate.pem.crt"
+        },
+        {
+            "name": "--key",
+            "data": "tests/ShadowUpdate/private.pem.key"
+        },
+        {
+            "name": "--thing_name",
+            "data": "ServiceTest_Shadow_$INPUT_UUID"
+        },
+        {
+            "name": "--shadow_property",
+            "data": "color"
+        },
+        {
+            "name": "--shadow_value",
+            "data": "on"
+        },
+        {
+            "name": "--shadow_name",
+            "data": "testShadow"
+        }
+    ]
+}
diff --git a/servicetests/test_cases/mqtt5_shadow_cfg.json b/servicetests/test_cases/mqtt5_shadow_cfg.json
@@ -0,0 +1,36 @@
+{
+    "language": "Python",
+    "runnable_file": "./tests/ShadowUpdate/shadow_update.py",
+    "runnable_region": "us-east-1",
+    "runnable_main_class": "",
+    "arguments": [
+        {
+            "name": "--mqtt_version",
+            "data": "5"
+        },
+        {
+            "name": "--endpoint",
+            "secret": "ci/endpoint"
+        },
+        {
+            "name": "--cert",
+            "data": "tests/ShadowUpdate/certificate.pem.crt"
+        },
+        {
+            "name": "--key",
+            "data": "tests/ShadowUpdate/private.pem.key"
+        },
+        {
+            "name": "--thing_name",
+            "data": "ServiceTest_Shadow_$INPUT_UUID"
+        },
+        {
+            "name": "--shadow_property",
+            "data": "color"
+        },
+        {
+            "name": "--shadow_value",
+            "data": "on"
+        }
+    ]
+}
diff --git a/servicetests/test_cases/test_shadow_update.py b/servicetests/test_cases/test_shadow_update.py
@@ -0,0 +1,128 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-3.0.
+
+import argparse
+import json
+import os
+import sys
+import uuid
+import time
+
+import boto3
+
+import run_in_ci
+import ci_iot_thing
+
+
+def get_shadow_attrs(config_file):
+    with open(config_file) as f:
+        json_data = json.load(f)
+        shadow_name = next((json_arg["data"] for json_arg in json_data["arguments"] if json_arg.get("name", "") == "--shadow_name"), "")
+        shadow_property = next((json_arg["data"] for json_arg in json_data["arguments"] if json_arg.get("name", "") == "--shadow_property"), "")
+        shadow_desired_value = next((json_arg["data"] for json_arg in json_data["arguments"] if json_arg.get("name", "") == "--shadow_value"), "")
+        return [shadow_name, shadow_property, shadow_desired_value]
+
+
+def main():
+    argument_parser = argparse.ArgumentParser(
+        description="Run Shadow test in CI")
+    argument_parser.add_argument(
+        "--config-file", required=True,
+        help="JSON file providing command-line arguments for a test")
+    argument_parser.add_argument(
+        "--input-uuid", required=False, help="UUID for thing name. UUID will be generated if this option is omit")
+    argument_parser.add_argument(
+        "--region", required=False, default="us-east-1", help="The name of the region to use")
+    parsed_commands = argument_parser.parse_args()
+
+    [shadow_name, shadow_property, shadow_desired_value] = get_shadow_attrs(parsed_commands.config_file)
+    print(f"Shadow name: '{shadow_name}'")
+    print(f"Shadow property: '{shadow_property}'")
+    print(f"Shadow desired value: '{shadow_desired_value}'")
+
+    try:
+        iot_data_client = boto3.client('iot-data', region_name=parsed_commands.region)
+        secrets_client = boto3.client("secretsmanager", region_name=parsed_commands.region)
+    except Exception as e:
+        print(f"ERROR: Could not make Boto3 iot-data client. Credentials likely could not be sourced. Exception: {e}",
+              file=sys.stderr)
+        return -1
+
+    input_uuid = parsed_commands.input_uuid if parsed_commands.input_uuid else str(uuid.uuid4())
+
+    thing_name = "ServiceTest_Shadow_" + input_uuid
+    policy_name = secrets_client.get_secret_value(
+        SecretId="ci/ShadowServiceClientTest/policy_name")["SecretString"]
+
+    # Temporary certificate/key file path.
+    certificate_path = os.path.join(os.getcwd(), "tests/ShadowUpdate/certificate.pem.crt")
+    key_path = os.path.join(os.getcwd(), "tests/ShadowUpdate/private.pem.key")
+
+    try:
+        ci_iot_thing.create_iot_thing(
+            thing_name=thing_name,
+            region=parsed_commands.region,
+            policy_name=policy_name,
+            certificate_path=certificate_path,
+            key_path=key_path)
+    except Exception as e:
+        print(f"ERROR: Failed to create IoT thing: {e}")
+        sys.exit(-1)
+
+    # Perform Shadow test. If it's successful, a shadow should appear for a specified thing.
+    try:
+        test_result = run_in_ci.setup_and_launch(parsed_commands.config_file, input_uuid)
+    except Exception as e:
+        print(f"ERROR: Failed to create shadow test: {e}")
+        test_result = -1
+
+    # Test reported success, verify that shadow was indeed updated.
+    if test_result == 0:
+        print("Verifying that shadow was updated")
+        shadow_value = None
+        try:
+            if shadow_name:
+                thing_shadow = iot_data_client.get_thing_shadow(thingName=thing_name, shadowName=shadow_name)
+            else:
+                thing_shadow = iot_data_client.get_thing_shadow(thingName=thing_name)
+
+            payload = thing_shadow['payload'].read()
+            data = json.loads(payload)
+            shadow_value = data.get('state', {}).get('reported', {}).get(shadow_property, None)
+            if shadow_value != shadow_desired_value:
+                print(f"ERROR: Could not verify thing shadow: {shadow_property} is not set to desired value "
+                      f"'{shadow_desired_value}'; shadow actual state: {data}")
+                test_result = -1
+        except KeyError as e:
+            print(f"ERROR: Could not verify thing shadow: key {e} does not exist in shadow response: {thing_shadow}")
+            test_result = -1
+        except Exception as e:
+            print(f"ERROR: Could not verify thing shadow: {e}")
+            test_result = -1
+
+    if test_result == 0:
+        print("Test succeeded")
+
+    # Delete a thing created for this test run.
+    # NOTE We want to try to delete thing even if test was unsuccessful.
+    try:
+        ci_iot_thing.delete_iot_thing(thing_name, parsed_commands.region)
+    except Exception as e:
+        print(f"ERROR: Failed to delete thing: {e}")
+        # Fail the test if unable to delete thing, so this won't remain unnoticed.
+        test_result = -1
+
+    try:
+        if os.path.isfile(certificate_path):
+            os.remove(certificate_path)
+        if os.path.isfile(key_path):
+            os.remove(key_path)
+    except Exception as e:
+        print(f"WARNING: Failed to delete local files: {e}")
+
+    if test_result != 0:
+        sys.exit(-1)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/servicetests/tests/JobsExecution/jobs.py b/servicetests/tests/JobsExecution/jobs.py
@@ -76,7 +76,6 @@ def exit(msg_or_exception):
             print("Disconnecting...")
             locked_data.disconnect_called = True
             if cmdData.input_mqtt_version == 5:
-                locked_data.disconnect_called = True
                 mqtt5_client.stop()
             else:
                 future = mqtt_connection.disconnect()
@@ -335,7 +334,6 @@ def on_lifecycle_stopped(lifecycle_stopped_data: mqtt5.LifecycleStoppedData):
         print("Unsopported MQTT version number\n")
         sys.exit(-1)
 
-
     print("Connected!")
 
     try:
diff --git a/servicetests/tests/ShadowUpdate/shadow_update.py b/servicetests/tests/ShadowUpdate/shadow_update.py
