Run Greengrass samples in CI (#585)

- Add tests for Greengrass samples: IPC and discovery.
- Add new CI job for Greengrass tests.

Author: sfod

.github/workflows/ci.yml

@@ -29,6 +29,7 @@ env:
   CI_JOBS_ROLE: arn:aws:iam::180635532705:role/CI_Jobs_Role
   CI_FLEET_PROVISIONING_ROLE: arn:aws:iam::180635532705:role/service-role/CI_FleetProvisioning_Role
   CI_GREENGRASS_ROLE: arn:aws:iam::180635532705:role/CI_Greengrass_Role
+  CI_GREENGRASS_INSTALLER_ROLE: arn:aws:iam::180635532705:role/CI_GreengrassInstaller_Role
   CI_DEVICE_ADVISOR: arn:aws:iam::180635532705:role/CI_DeviceAdvisor_Role
   CI_MQTT5_ROLE: arn:aws:iam::180635532705:role/CI_MQTT5_Role
   CI_BUILD_AND_TEST_ROLE: arn:aws:iam::180635532705:role/V2_SDK_Unit_Testing
@@ -393,14 +394,66 @@ jobs:
           Sample_UUID=$(python3  -c "import uuid; print (uuid.uuid4())")
           python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --file ${{ env.CI_SAMPLES_CFG_FOLDER }}/ci_run_mqtt5_fleet_provisioning_cfg.json --input_uuid ${Sample_UUID}
           python3 ${{ env.CI_UTILS_FOLDER }}/delete_iot_thing_ci.py --thing_name "Fleet_Thing_${Sample_UUID}" --region "us-east-1"
-      - name: configure AWS credentials (Greengrass)
+
+  linux-greengrass-tests:
+    runs-on: ubuntu-22.04
+    permissions:
+      id-token: write # This is required for requesting the JWT
+    steps:
+      - name: Setup environment
+        run: |
+          python3 -m pip install boto3
+      - name: configure AWS credentials (containers)
         uses: aws-actions/configure-aws-credentials@v2
         with:
-          role-to-assume: ${{ env.CI_GREENGRASS_ROLE }}
+          role-to-assume: ${{ env.CI_BUILD_AND_TEST_ROLE }}
           aws-region: ${{ env.AWS_DEFAULT_REGION }}
-      - name: run Greengrass Discovery sample
+      - name: Build ${{ env.PACKAGE_NAME }}
+        run: |
+          python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')"
+          chmod a+x builder
+          ./builder build -p ${{ env.PACKAGE_NAME }}
+      - name: Install Greengrass Development Kit
         run: |
-          python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --file ${{ env.CI_SAMPLES_CFG_FOLDER }}/ci_run_greengrass_discovery_cfg.json
+          python3 -m pip install awsiotsdk
+          python3 -m pip install -U git+https://github.com/aws-greengrass/[email protected]
+      - name: Configure AWS credentials (Greengrass)
+        uses: aws-actions/configure-aws-credentials@v2
+        with:
+          role-to-assume: ${{ env.CI_GREENGRASS_INSTALLER_ROLE }}
+          aws-region: ${{ env.AWS_DEFAULT_REGION }}
+      - name: Build and run Greengrass basic discovery sample
+        working-directory: ./aws-iot-device-sdk-python-v2/test/greengrass/basic_discovery
+        run: |
+          export PYTHONPATH=${{ github.workspace }}/aws-iot-device-sdk-python-v2/samples
+          gdk component build
+          gdk test-e2e build
+          gdk test-e2e run
+      - name: Show logs
+        working-directory: ./aws-iot-device-sdk-python-v2/test/greengrass/basic_discovery
+        # Print logs unconditionally to provide more details on Greengrass run even if the test failed.
+        if: always()
+        run: |
+          echo "=== greengrass.log"
+          cat testResults/gg*/greengrass.log
+          echo "=== software.amazon.awssdk.sdk-gg-test-discovery.log"
+          cat testResults/gg*/software.amazon.awssdk.sdk-gg-test-discovery.log
+      - name: Build and run Greengrass IPC sample
+        working-directory: ./aws-iot-device-sdk-python-v2/test/greengrass/ipc
+        run: |
+          gdk component build
+          gdk test-e2e build
+          gdk test-e2e run
+      - name: Show logs
+        working-directory: ./aws-iot-device-sdk-python-v2/test/greengrass/ipc
+        # Print logs unconditionally to provide more details on Greengrass run even if the test failed.
+        if: always()
+        run: |
+          echo "=== greengrass.log"
+          cat testResults/gg*/greengrass.log
+          echo "=== software.amazon.awssdk.sdk-gg-ipc.log"
+          cat testResults/gg*/software.amazon.awssdk.sdk-gg-ipc.log
+
 
   # check that docs can still build
   check-docs:

.github/workflows/ci_run_basic_connect_cfg.json

@@ -17,6 +17,10 @@
             "name": "--key",
             "secret": "ci/PubSub/key",
             "filename": "tmp_key.pem"
+        },
+        {
+            "name": "--is_ci",
+            "data": "true"
         }
     ]
 }

.github/workflows/ci_run_cognito_connect_cfg.json

@@ -15,6 +15,10 @@
         {
             "name": "--cognito_identity",
             "secret": "ci/Cognito/identity_id"
+        },
+        {
+            "name": "--is_ci",
+            "data": "true"
         }
     ]
 }

.github/workflows/ci_run_custom_authorizer_connect_cfg.json

@@ -15,6 +15,10 @@
         {
             "name": "--custom_auth_password",
             "secret": "ci/CustomAuthorizer/password"
+        },
+        {
+            "name": "--is_ci",
+            "data": "true"
         }
     ]
 }

.github/workflows/ci_run_fleet_provisioning_cfg.json

@@ -25,6 +25,10 @@
         {
             "name": "--template_parameters",
             "data": "{\"SerialNumber\":\"$INPUT_UUID\"}"
+        },
+        {
+            "name": "--is_ci",
+            "data": "true"
         }
     ]
 }

.github/workflows/ci_run_greengrass_discovery_cfg.json

@@ -1,39 +1,34 @@
 {
     "language": "Python",
-    "sample_file": "./aws-iot-device-sdk-python-v2/samples/basic_discovery.py",
-    "sample_region": "us-east-1",
-    "sample_main_class": "",
+    "runnable_file": "basic_discovery.py",
+    "runnable_region": "us-east-1",
+    "runnable_main_class": "",
     "arguments": [
         {
             "name": "--cert",
-            "secret": "ci/Greengrass/cert",
+            "secret": "ci/GreengrassDiscovery/cert",
             "filename": "tmp_certificate.pem"
         },
         {
             "name": "--key",
-            "secret": "ci/Greengrass/key",
+            "secret": "ci/GreengrassDiscovery/key",
             "filename": "tmp_key.pem"
         },
         {
-            "name": "--ca_file",
-            "secret": "ci/Greengrass/ca",
-            "filename": "tmp_ca.pem"
+            "name": "--thing_name",
+            "data": "CI_Greengrass_Discovery_Thing"
         },
         {
             "name": "--region",
             "data": "us-east-1"
         },
         {
-            "name": "--thing_name",
-            "data": "CI_GreenGrass_Thing"
-        },
-        {
-            "name": "--is_ci",
-            "data": "true"
+            "name": "--topic",
+            "data": "clients/CI_Greengrass_Discovery_Thing/hello/world/$INPUT_UUID"
         },
         {
-            "name": "--print_discover_resp_only",
-            "data": ""
+            "name": "--mode",
+            "data": "publish"
         }
     ]
 }

.github/workflows/ci_run_greengrass_ipc_cfg.json

@@ -0,0 +1,20 @@
+{
+    "language": "Python",
+    "runnable_file": "ipc_greengrass.py",
+    "runnable_region": "us-east-1",
+    "runnable_main_class": "",
+    "arguments": [
+        {
+            "name": "--topic",
+            "data": "test/gg-ipc-topic"
+        },
+        {
+            "name": "--message",
+            "data": "hello"
+        },
+        {
+            "name": "--is_ci",
+            "data": "true"
+        }
+    ]
+}

.github/workflows/ci_run_jobs_cfg.json

@@ -21,6 +21,10 @@
         {
             "name": "--thing_name",
             "data": "CI_Jobs_Thing"
+        },
+        {
+            "name": "--is_ci",
+            "data": "true"
         }
     ]
 }

.github/workflows/ci_run_mqtt5_custom_authorizer_cfg.json

@@ -15,6 +15,10 @@
         {
             "name": "--custom_auth_password",
             "secret": "ci/CustomAuthorizer/password"
+        },
+        {
+            "name": "--is_ci",
+            "data": "true"
         }
     ]
 }

.github/workflows/ci_run_mqtt5_custom_authorizer_websockets_cfg.json

@@ -19,6 +19,10 @@
         {
             "name": "--use_websockets",
             "data": "true"
+        },
+        {
+            "name": "--is_ci",
+            "data": "true"
         }
     ]
 }

.github/workflows/ci_run_mqtt5_fleet_provisioning_cfg.json

@@ -25,6 +25,10 @@
         {
             "name": "--template_parameters",
             "data": "{\"SerialNumber\":\"$INPUT_UUID\"}"
+        },
+        {
+            "name": "--is_ci",
+            "data": "true"
         }
     ]
 }

.github/workflows/ci_run_mqtt5_jobs_cfg.json

@@ -21,6 +21,10 @@
         {
             "name": "--thing_name",
             "data": "CI_Jobs_Thing"
+        },
+        {
+            "name": "--is_ci",
+            "data": "true"
         }
     ]
 }

.github/workflows/ci_run_mqtt5_pkcs11_connect_cfg.json

@@ -34,6 +34,10 @@
         {
             "name": "--key_label",
             "data": "my-key"
+        },
+        {
+            "name": "--is_ci",
+            "data": "true"
         }
     ]
 }

.github/workflows/ci_run_pkcs11_connect_cfg.json

@@ -34,6 +34,10 @@
         {
             "name": "--key_label",
             "data": "my-key"
+        },
+        {
+            "name": "--is_ci",
+            "data": "true"
         }
     ]
 }

.github/workflows/ci_run_pkcs12_connect_cfg.json

@@ -15,6 +15,10 @@
     {
         "name": "--pkcs12_password",
         "secret": "ci/PubSub/key_pkcs12_password"
+    },
+    {
+        "name": "--is_ci",
+        "data": "true"
     }
 ]
 }

.github/workflows/ci_run_websocket_connect_cfg.json

@@ -11,6 +11,10 @@
         {
             "name": "--signing_region",
             "data": "us-east-1"
+    },
+    {
+        "name": "--is_ci",
+        "data": "true"
         }
     ]
 }

.github/workflows/ci_run_windows_cert_connect_cfg.json

@@ -15,6 +15,10 @@
             "windows_cert_key": "ci/PubSub/key",
             "windows_cert_key_path": "tmp_key.pem",
             "windows_cert_pfx_key_path": "tmp_pfx.pem"
+        },
+        {
+            "name": "--is_ci",
+            "data": "true"
         }
     ]
 }

.github/workflows/ci_run_x509_connect_cfg.json

@@ -33,6 +33,10 @@
         {
             "name": "--x509_thing_name",
             "data": "CI_PubSub_Thing"
+        },
+        {
+            "name": "--is_ci",
+            "data": "true"
         }
     ]
 }

samples/basic_discovery.py

@@ -102,9 +102,9 @@ def on_publish(topic, payload, dup, qos, retain, **kwargs):
         message['message'] = cmdData.input_message
         message['sequence'] = loop_count
         messageJson = json.dumps(message)
-        pub_future, _ = mqtt_connection.publish(cmdData.input_topic, messageJson, QoS.AT_MOST_ONCE)
-        pub_future.result()
-        print('Published topic {}: {}\n'.format(cmdData.input_topic, messageJson))
+        pub_future, _ = mqtt_connection.publish(cmdData.input_topic, messageJson, QoS.AT_LEAST_ONCE)
+        publish_completion_data = pub_future.result()
+        print('Published topic {}: {} (puback reason: {})\n'.format(cmdData.input_topic, messageJson, repr(publish_completion_data.puback.reason_code)))
 
         loop_count += 1
     time.sleep(1)

samples/ipc_greengrass.py

@@ -22,33 +22,39 @@
 import json
 import time
 import os
+import sys
 
 import awsiot.greengrasscoreipc
 import awsiot.greengrasscoreipc.model as model
 
 if __name__ == '__main__':
-    ipc_client = awsiot.greengrasscoreipc.connect()
-
-    while True:
-        telemetry_data = {
-            "timestamp": int(round(time.time() * 1000)),
-            "battery_state_of_charge": 42.5,
-            "location": {
-                "longitude": 48.15743,
-                "latitude": 11.57549,
-            },
-        }
-
-        op = ipc_client.new_publish_to_iot_core()
-        op.activate(model.PublishToIoTCoreRequest(
-            topic_name="my/iot/{}/telemetry".format(os.getenv("AWS_IOT_THING_NAME")),
-            qos=model.QOS.AT_LEAST_ONCE,
-            payload=json.dumps(telemetry_data).encode(),
-        ))
-        try:
-            result = op.get_response().result(timeout=5.0)
-            print("successfully published message:", result)
-        except Exception as e:
-            print("failed to publish message:", e)
-
-        time.sleep(5)
+    try:
+        print("Connecting to Greengrass...")
+        ipc_client = awsiot.greengrasscoreipc.connect()
+
+        for i in range(0, 10):
+            telemetry_data = {
+                "timestamp": int(round(time.time() * 1000)),
+                "battery_state_of_charge": 42.5,
+                "location": {
+                    "longitude": 48.15743,
+                    "latitude": 11.57549,
+                },
+            }
+
+            op = ipc_client.new_publish_to_iot_core()
+            op.activate(model.PublishToIoTCoreRequest(
+                topic_name="my/iot/{}/telemetry".format(os.getenv("AWS_IOT_THING_NAME")),
+                qos=model.QOS.AT_LEAST_ONCE,
+                payload=json.dumps(telemetry_data).encode(),
+            ))
+            try:
+                result = op.get_response().result(timeout=5.0)
+                print("Successfully published message: {}".format(result))
+            except Exception as e:
+                print("Failed to publish message: {}".format(e))
+                sys.exit(1)
+
+    except Exception as e:
+        print("Sample failed: {}".format(e))
+        sys.exit(1)