From ba9f4d042e332f9bc95b740376162f03acabfebd Mon Sep 17 00:00:00 2001 From: Igor Abdrakhimov Date: Tue, 2 Jan 2024 14:17:29 -0800 Subject: [PATCH 1/5] Disable shadow tests --- .github/workflows/ci.yml | 51 ++++++++++++++++++++-------------------- 1 file changed, 26 insertions(+), 25 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a1076514..4a917ccb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -234,31 +234,32 @@ jobs: export PYTHONPATH=${{ github.workspace }}/aws-iot-device-sdk-python-v2/utils:${{ github.workspace }}/aws-iot-device-sdk-python-v2/samples python3 ./test_cases/test_fleet_provisioning.py --config-file test_cases/mqtt5_fleet_provisioning_with_csr_cfg.json --thing-name-prefix Fleet_Thing_ - - name: configure AWS credentials (service tests Shadow) - uses: aws-actions/configure-aws-credentials@v2 - with: - role-to-assume: ${{ env.CI_SHADOW_SERVICE_CLIENT_ROLE }} - aws-region: ${{ env.AWS_DEFAULT_REGION }} - - name: run MQTT5 Shadow Update - working-directory: ./aws-iot-device-sdk-python-v2/servicetests - run: | - export PYTHONPATH=${{ github.workspace }}/aws-iot-device-sdk-python-v2/utils:${{ github.workspace }}/aws-iot-device-sdk-python-v2/samples - python3 ./test_cases/test_shadow_update.py --config-file test_cases/mqtt5_shadow_cfg.json - - name: run MQTT3 Shadow Update - working-directory: ./aws-iot-device-sdk-python-v2/servicetests - run: | - export PYTHONPATH=${{ github.workspace }}/aws-iot-device-sdk-python-v2/utils:${{ github.workspace }}/aws-iot-device-sdk-python-v2/samples - python3 ./test_cases/test_shadow_update.py --config-file test_cases/mqtt3_shadow_cfg.json - - name: run MQTT5 Named Shadow Update - working-directory: ./aws-iot-device-sdk-python-v2/servicetests - run: | - export PYTHONPATH=${{ github.workspace }}/aws-iot-device-sdk-python-v2/utils:${{ github.workspace }}/aws-iot-device-sdk-python-v2/samples - python3 ./test_cases/test_shadow_update.py --config-file test_cases/mqtt5_named_shadow_cfg.json - - name: run MQTT3 Named Shadow Update - working-directory: ./aws-iot-device-sdk-python-v2/servicetests - run: | - export PYTHONPATH=${{ github.workspace }}/aws-iot-device-sdk-python-v2/utils:${{ github.workspace }}/aws-iot-device-sdk-python-v2/samples - python3 ./test_cases/test_shadow_update.py --config-file test_cases/mqtt3_named_shadow_cfg.json + # TODO Add waiting logic to checks, only then reenable these tests. + # - name: configure AWS credentials (service tests Shadow) + # uses: aws-actions/configure-aws-credentials@v2 + # with: + # role-to-assume: ${{ env.CI_SHADOW_SERVICE_CLIENT_ROLE }} + # aws-region: ${{ env.AWS_DEFAULT_REGION }} + # - name: run MQTT5 Shadow Update + # working-directory: ./aws-iot-device-sdk-python-v2/servicetests + # run: | + # export PYTHONPATH=${{ github.workspace }}/aws-iot-device-sdk-python-v2/utils:${{ github.workspace }}/aws-iot-device-sdk-python-v2/samples + # python3 ./test_cases/test_shadow_update.py --config-file test_cases/mqtt5_shadow_cfg.json + # - name: run MQTT3 Shadow Update + # working-directory: ./aws-iot-device-sdk-python-v2/servicetests + # run: | + # export PYTHONPATH=${{ github.workspace }}/aws-iot-device-sdk-python-v2/utils:${{ github.workspace }}/aws-iot-device-sdk-python-v2/samples + # python3 ./test_cases/test_shadow_update.py --config-file test_cases/mqtt3_shadow_cfg.json + # - name: run MQTT5 Named Shadow Update + # working-directory: ./aws-iot-device-sdk-python-v2/servicetests + # run: | + # export PYTHONPATH=${{ github.workspace }}/aws-iot-device-sdk-python-v2/utils:${{ github.workspace }}/aws-iot-device-sdk-python-v2/samples + # python3 ./test_cases/test_shadow_update.py --config-file test_cases/mqtt5_named_shadow_cfg.json + # - name: run MQTT3 Named Shadow Update + # working-directory: ./aws-iot-device-sdk-python-v2/servicetests + # run: | + # export PYTHONPATH=${{ github.workspace }}/aws-iot-device-sdk-python-v2/utils:${{ github.workspace }}/aws-iot-device-sdk-python-v2/samples + # python3 ./test_cases/test_shadow_update.py --config-file test_cases/mqtt3_named_shadow_cfg.json - name: configure AWS credentials (service tests Jobs) uses: aws-actions/configure-aws-credentials@v2 From 77e1bb6a52f271f521e2f7e13291441b538a2c6d Mon Sep 17 00:00:00 2001 From: Igor Abdrakhimov Date: Tue, 2 Jan 2024 14:19:55 -0800 Subject: [PATCH 2/5] Enable shadow tests --- .github/workflows/ci.yml | 51 ++++++++++++++++++++-------------------- 1 file changed, 25 insertions(+), 26 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4a917ccb..a1076514 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -234,32 +234,31 @@ jobs: export PYTHONPATH=${{ github.workspace }}/aws-iot-device-sdk-python-v2/utils:${{ github.workspace }}/aws-iot-device-sdk-python-v2/samples python3 ./test_cases/test_fleet_provisioning.py --config-file test_cases/mqtt5_fleet_provisioning_with_csr_cfg.json --thing-name-prefix Fleet_Thing_ - # TODO Add waiting logic to checks, only then reenable these tests. - # - name: configure AWS credentials (service tests Shadow) - # uses: aws-actions/configure-aws-credentials@v2 - # with: - # role-to-assume: ${{ env.CI_SHADOW_SERVICE_CLIENT_ROLE }} - # aws-region: ${{ env.AWS_DEFAULT_REGION }} - # - name: run MQTT5 Shadow Update - # working-directory: ./aws-iot-device-sdk-python-v2/servicetests - # run: | - # export PYTHONPATH=${{ github.workspace }}/aws-iot-device-sdk-python-v2/utils:${{ github.workspace }}/aws-iot-device-sdk-python-v2/samples - # python3 ./test_cases/test_shadow_update.py --config-file test_cases/mqtt5_shadow_cfg.json - # - name: run MQTT3 Shadow Update - # working-directory: ./aws-iot-device-sdk-python-v2/servicetests - # run: | - # export PYTHONPATH=${{ github.workspace }}/aws-iot-device-sdk-python-v2/utils:${{ github.workspace }}/aws-iot-device-sdk-python-v2/samples - # python3 ./test_cases/test_shadow_update.py --config-file test_cases/mqtt3_shadow_cfg.json - # - name: run MQTT5 Named Shadow Update - # working-directory: ./aws-iot-device-sdk-python-v2/servicetests - # run: | - # export PYTHONPATH=${{ github.workspace }}/aws-iot-device-sdk-python-v2/utils:${{ github.workspace }}/aws-iot-device-sdk-python-v2/samples - # python3 ./test_cases/test_shadow_update.py --config-file test_cases/mqtt5_named_shadow_cfg.json - # - name: run MQTT3 Named Shadow Update - # working-directory: ./aws-iot-device-sdk-python-v2/servicetests - # run: | - # export PYTHONPATH=${{ github.workspace }}/aws-iot-device-sdk-python-v2/utils:${{ github.workspace }}/aws-iot-device-sdk-python-v2/samples - # python3 ./test_cases/test_shadow_update.py --config-file test_cases/mqtt3_named_shadow_cfg.json + - name: configure AWS credentials (service tests Shadow) + uses: aws-actions/configure-aws-credentials@v2 + with: + role-to-assume: ${{ env.CI_SHADOW_SERVICE_CLIENT_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} + - name: run MQTT5 Shadow Update + working-directory: ./aws-iot-device-sdk-python-v2/servicetests + run: | + export PYTHONPATH=${{ github.workspace }}/aws-iot-device-sdk-python-v2/utils:${{ github.workspace }}/aws-iot-device-sdk-python-v2/samples + python3 ./test_cases/test_shadow_update.py --config-file test_cases/mqtt5_shadow_cfg.json + - name: run MQTT3 Shadow Update + working-directory: ./aws-iot-device-sdk-python-v2/servicetests + run: | + export PYTHONPATH=${{ github.workspace }}/aws-iot-device-sdk-python-v2/utils:${{ github.workspace }}/aws-iot-device-sdk-python-v2/samples + python3 ./test_cases/test_shadow_update.py --config-file test_cases/mqtt3_shadow_cfg.json + - name: run MQTT5 Named Shadow Update + working-directory: ./aws-iot-device-sdk-python-v2/servicetests + run: | + export PYTHONPATH=${{ github.workspace }}/aws-iot-device-sdk-python-v2/utils:${{ github.workspace }}/aws-iot-device-sdk-python-v2/samples + python3 ./test_cases/test_shadow_update.py --config-file test_cases/mqtt5_named_shadow_cfg.json + - name: run MQTT3 Named Shadow Update + working-directory: ./aws-iot-device-sdk-python-v2/servicetests + run: | + export PYTHONPATH=${{ github.workspace }}/aws-iot-device-sdk-python-v2/utils:${{ github.workspace }}/aws-iot-device-sdk-python-v2/samples + python3 ./test_cases/test_shadow_update.py --config-file test_cases/mqtt3_named_shadow_cfg.json - name: configure AWS credentials (service tests Jobs) uses: aws-actions/configure-aws-credentials@v2 From 0249f990be49993348b8d57898b144b08762e158 Mon Sep 17 00:00:00 2001 From: Igor Abdrakhimov Date: Tue, 2 Jan 2024 14:27:39 -0800 Subject: [PATCH 3/5] Add waiting logic for shadow test --- servicetests/test_cases/test_shadow_update.py | 44 +++++++++++-------- 1 file changed, 26 insertions(+), 18 deletions(-) diff --git a/servicetests/test_cases/test_shadow_update.py b/servicetests/test_cases/test_shadow_update.py index 46c71ef6..16677a29 100644 --- a/servicetests/test_cases/test_shadow_update.py +++ b/servicetests/test_cases/test_shadow_update.py @@ -80,25 +80,33 @@ def main(): if test_result == 0: print("Verifying that shadow was updated") shadow_value = None - try: - if shadow_name: - thing_shadow = iot_data_client.get_thing_shadow(thingName=thing_name, shadowName=shadow_name) - else: - thing_shadow = iot_data_client.get_thing_shadow(thingName=thing_name) - - payload = thing_shadow['payload'].read() - data = json.loads(payload) - shadow_value = data.get('state', {}).get('reported', {}).get(shadow_property, None) - if shadow_value != shadow_desired_value: - print(f"ERROR: Could not verify thing shadow: {shadow_property} is not set to desired value " - f"'{shadow_desired_value}'; shadow actual state: {data}") + i = 0 + while i < 10: + try: + if shadow_name: + thing_shadow = iot_data_client.get_thing_shadow(thingName=thing_name, shadowName=shadow_name) + else: + thing_shadow = iot_data_client.get_thing_shadow(thingName=thing_name) + + payload = thing_shadow['payload'].read() + data = json.loads(payload) + shadow_value = data.get('state', {}).get('reported', {}).get(shadow_property, None) + if shadow_value == shadow_desired_value: + test_result = 0 + break + else: + print(f"ERROR: Could not verify thing shadow: {shadow_property} is not set to desired value " + f"'{shadow_desired_value}'; shadow actual state: {data}") + test_result = -1 + except KeyError as e: + print(f"ERROR: Could not verify thing shadow: key {e} does not exist in shadow response: {thing_shadow}") test_result = -1 - except KeyError as e: - print(f"ERROR: Could not verify thing shadow: key {e} does not exist in shadow response: {thing_shadow}") - test_result = -1 - except Exception as e: - print(f"ERROR: Could not verify thing shadow: {e}") - test_result = -1 + except Exception as e: + print(f"ERROR: Could not verify thing shadow: {e}") + test_result = -1 + i = i + 1 + time.sleep(1); + if test_result == 0: print("Test succeeded") From d372b120f6083f189064f8afa2f3db4108f3731d Mon Sep 17 00:00:00 2001 From: Igor Abdrakhimov Date: Tue, 2 Jan 2024 14:30:52 -0800 Subject: [PATCH 4/5] Increase sleep time --- servicetests/test_cases/test_shadow_update.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/servicetests/test_cases/test_shadow_update.py b/servicetests/test_cases/test_shadow_update.py index 16677a29..5b62557d 100644 --- a/servicetests/test_cases/test_shadow_update.py +++ b/servicetests/test_cases/test_shadow_update.py @@ -81,7 +81,7 @@ def main(): print("Verifying that shadow was updated") shadow_value = None i = 0 - while i < 10: + while i < 20: try: if shadow_name: thing_shadow = iot_data_client.get_thing_shadow(thingName=thing_name, shadowName=shadow_name) @@ -105,7 +105,7 @@ def main(): print(f"ERROR: Could not verify thing shadow: {e}") test_result = -1 i = i + 1 - time.sleep(1); + time.sleep(3); if test_result == 0: From b45e25ea9337b54ab0c22f04ee5604d0282df698 Mon Sep 17 00:00:00 2001 From: Igor Abdrakhimov Date: Tue, 2 Jan 2024 14:45:37 -0800 Subject: [PATCH 5/5] Fix race condition --- servicetests/test_cases/test_shadow_update.py | 2 +- .../tests/ShadowUpdate/shadow_update.py | 43 +------------------ 2 files changed, 3 insertions(+), 42 deletions(-) diff --git a/servicetests/test_cases/test_shadow_update.py b/servicetests/test_cases/test_shadow_update.py index 5b62557d..3bd6b774 100644 --- a/servicetests/test_cases/test_shadow_update.py +++ b/servicetests/test_cases/test_shadow_update.py @@ -81,7 +81,7 @@ def main(): print("Verifying that shadow was updated") shadow_value = None i = 0 - while i < 20: + while i < 10: try: if shadow_name: thing_shadow = iot_data_client.get_thing_shadow(thingName=thing_name, shadowName=shadow_name) diff --git a/servicetests/tests/ShadowUpdate/shadow_update.py b/servicetests/tests/ShadowUpdate/shadow_update.py index 00d5116d..8c89cb2d 100644 --- a/servicetests/tests/ShadowUpdate/shadow_update.py +++ b/servicetests/tests/ShadowUpdate/shadow_update.py @@ -100,7 +100,6 @@ def on_get_shadow_accepted(response): value = response.state.delta.get(shadow_property) if value: print(" Shadow contains delta value '{}'.".format(value)) - change_shadow_value(value) return if response.state.reported: @@ -108,9 +107,6 @@ def on_get_shadow_accepted(response): if value: print(" Shadow contains reported value '{}'.".format(value)) return - - print(" Shadow document lacks '{}' property. Setting defaults...".format(shadow_property)) - change_shadow_value(SHADOW_VALUE_DEFAULT) return except Exception as e: @@ -128,12 +124,8 @@ def on_get_shadow_rejected(error): print("Ignoring get_shadow_rejected message due to unexpected token.") return - if error.code == 404: - print("Thing has no shadow document. Creating with defaults...") - change_shadow_value(SHADOW_VALUE_DEFAULT) - else: - exit("Get request was rejected. code:{} message:'{}'".format( - error.code, error.message)) + if error.code != 404: + exit("Get request was rejected. code:{} message:'{}'".format(error.code, error.message)) except Exception as e: exit(e) @@ -193,32 +185,6 @@ def on_update_shadow_rejected(error): exit(e) -def change_shadow_value(value): - with locked_data.lock: - - print("Changed local shadow value to '{}'.".format(value)) - locked_data.shadow_value = value - - print("Updating reported shadow value to '{}'...".format(value)) - - # use a unique token so we can correlate this "request" message to - # any "response" messages received on the /accepted and /rejected topics - token = str(uuid4()) - - # if the value is "none" then set it to a Python none object to - request = iotshadow.UpdateShadowRequest( - thing_name=shadow_thing_name, - state=iotshadow.ShadowState( - reported={shadow_property: value}, - desired={shadow_property: value}, - ), - client_token=token, - ) - future = shadow_client.publish_update_shadow(request, mqtt_qos) - locked_data.request_tokens.add(token) - future.add_done_callback(on_publish_update_shadow) - - def update_event_received(response): print("Update Event Received\n") print("Current response", response.current) @@ -387,8 +353,6 @@ def update_shadow(): update_thing_update_future = shadow_client.publish_update_shadow(request = iotshadow.UpdateShadowRequest (thing_name = shadow_thing_name, state=state), qos=mqtt_qos) - - change_shadow_value(cmdData.input_shadow_value) update_thing_update_future.result() except Exception as e: @@ -470,6 +434,3 @@ def update_shadow(): exit(0) # Wait for the sample to finish is_sample_done.wait() - - -