Sample and command line refactor (#441)

Refactor samples to not rely on cmdUtils to make the actual MQTT connection, adjust command line parsing to return single class/struct with all the data set, and minor fixes.

Author: TwistedTwigleg

.github/workflows/ci_run_custom_authorizer_connect_cfg.json

@@ -8,6 +8,10 @@
             "name": "--endpoint",
             "secret": "ci/endpoint"
         },
+        {
+            "name": "--signing_region",
+            "data": "us-east-1"
+        },
         {
             "name": "--custom_auth_authorizer_name",
             "secret": "ci/CustomAuthorizer/name"

.github/workflows/ci_run_mqtt5_custom_authorizer_cfg.json

@@ -15,6 +15,16 @@
         {
             "name": "--custom_auth_password",
             "secret": "ci/CustomAuthorizer/password"
+        },
+        {
+            "name": "--cert",
+            "secret": "ci/mqtt5/us/mqtt5_thing/cert",
+            "filename": "tmp_certificate.pem"
+        },
+        {
+            "name": "--key",
+            "secret": "ci/mqtt5/us/mqtt5_thing/key",
+            "filename": "tmp_key.pem"
         }
     ]
 }

.github/workflows/ci_run_mqtt5_custom_authorizer_websockets_cfg.json

@@ -19,6 +19,10 @@
         {
             "name": "--use_websockets",
             "data": "true"
+        },
+        {
+            "name": "--signing_region",
+            "data": "us-east-1"
         }
     ]
 }

.github/workflows/ci_run_mqtt5_pubsub_cfg.json

@@ -17,6 +17,10 @@
             "name": "--key",
             "secret": "ci/mqtt5/us/mqtt5_thing/key",
             "filename": "tmp_key.pem"
+        },
+        {
+            "name": "--is_ci",
+            "data": "true"
         }
     ]
 }

.github/workflows/ci_run_mqtt5_shared_subscription_cfg.json

@@ -17,6 +17,10 @@
             "name": "--key",
             "secret": "ci/mqtt5/us/mqtt5_thing/key",
             "filename": "tmp_key.pem"
+        },
+        {
+            "name": "--is_ci",
+            "data": "true"
         }
     ]
 }

.github/workflows/ci_run_pubsub_cfg.json

@@ -17,6 +17,10 @@
             "name": "--key",
             "secret": "ci/PubSub/key",
             "filename": "tmp_key.pem"
+        },
+        {
+            "name": "--is_ci",
+            "data": "true"
         }
     ]
 }

codebuild/samples/custom-auth-linux.sh

@@ -12,6 +12,6 @@ AUTH_NAME=$(aws secretsmanager get-secret-value --secret-id "ci/CustomAuthorizer
 AUTH_PASSWORD=$(aws secretsmanager get-secret-value --secret-id "ci/CustomAuthorizer/password" --query "SecretString" | cut -f2 -d":" | sed -e 's/[\\\"\}]//g')
 
 echo "Custom Authorizer test"
-python3 custom_authorizer_connect.py --endpoint $ENDPOINT --custom_auth_authorizer_name $AUTH_NAME --custom_auth_password $AUTH_PASSWORD
+python3 custom_authorizer_connect.py --endpoint $ENDPOINT --custom_auth_authorizer_name $AUTH_NAME --custom_auth_password $AUTH_PASSWORD --signing_region us-east-1
 
 popd

codebuild/samples/pubsub-linux.sh

@@ -10,6 +10,6 @@ pushd $CODEBUILD_SRC_DIR/samples/
 ENDPOINT=$(aws secretsmanager get-secret-value --secret-id "ci/endpoint" --query "SecretString" | cut -f2 -d":" | sed -e 's/[\\\"\}]//g')
 
 echo "PubSub test"
-python3 pubsub.py --endpoint $ENDPOINT --key /tmp/privatekey.pem --cert /tmp/certificate.pem
+python3 pubsub.py --endpoint $ENDPOINT --key /tmp/privatekey.pem --cert /tmp/certificate.pem --is_ci "true"
 
 popd

codebuild/samples/pubsub-mqtt5-linux.sh

@@ -10,6 +10,6 @@ pushd $CODEBUILD_SRC_DIR/samples/
 ENDPOINT=$(aws secretsmanager get-secret-value --secret-id "ci/endpoint" --query "SecretString" | cut -f2 -d":" | sed -e 's/[\\\"\}]//g')
 
 echo "MQTT5 PubSub test"
-python3 mqtt5_pubsub.py --endpoint $ENDPOINT --key /tmp/privatekey.pem --cert /tmp/certificate.pem
+python3 mqtt5_pubsub.py --endpoint $ENDPOINT --key /tmp/privatekey.pem --cert /tmp/certificate.pem --is_ci "true"
 
 popd

samples/basic_connect.py

@@ -1,31 +1,13 @@
 # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 # SPDX-License-Identifier: Apache-2.0.
 
-from uuid import uuid4
+from awscrt import http
+from awsiot import mqtt_connection_builder
+from utils.command_line_utils import CommandLineUtils
 
 # This sample shows how to create a MQTT connection using a certificate file and key file.
 # This sample is intended to be used as a reference for making MQTT connections.
 
-# Parse arguments
-import utils.command_line_utils as command_line_utils
-cmdUtils = command_line_utils.CommandLineUtils("Basic Connect - Make a MQTT connection.")
-cmdUtils.add_common_mqtt_commands()
-cmdUtils.add_common_proxy_commands()
-cmdUtils.add_common_logging_commands()
-cmdUtils.register_command("key", "<path>", "Path to your key in PEM format.", True, str)
-cmdUtils.register_command("cert", "<path>", "Path to your client certificate in PEM format.", True, str)
-cmdUtils.register_command("port", "<int>",
-                          "Connection port for direct connection. " +
-                          "AWS IoT supports 443 and 8883 (optional, default=8883).",
-                          False, int)
-cmdUtils.register_command("client_id", "<str>",
-                          "Client ID to use for MQTT connection (optional, default='test-*').",
-                          default="test-" + str(uuid4()))
-cmdUtils.register_command("is_ci", "<str>", "If present the sample will run in CI mode (optional, default='None')")
-# Needs to be called so the command utils parse the commands
-cmdUtils.get_args()
-is_ci = cmdUtils.get_command("is_ci", None) != None
-
 # Callback when connection is accidentally lost.
 def on_connection_interrupted(connection, error, **kwargs):
     print("Connection interrupted. error: {}".format(error))
@@ -36,19 +18,39 @@ def on_connection_resumed(connection, return_code, session_present, **kwargs):
 
 
 if __name__ == '__main__':
-    # Create a connection using a certificate and key.
-    # Note: The data for the connection is gotten from cmdUtils.
-    # (see build_direct_mqtt_connection for implementation)
-    mqtt_connection = cmdUtils.build_direct_mqtt_connection(on_connection_interrupted, on_connection_resumed)
-
-    if is_ci == False:
-        print("Connecting to {} with client ID '{}'...".format(
-            cmdUtils.get_command(cmdUtils.m_cmd_endpoint), cmdUtils.get_command("client_id")))
+
+    # cmdData is the arguments/input from the command line placed into a single struct for
+    # use in this sample. This handles all of the command line parsing, validating, etc.
+    # See the Utils/CommandLineUtils for more information.
+    cmdData = CommandLineUtils.parse_sample_input_basic_connect()
+
+    # Create the proxy options if the data is present in cmdData
+    proxy_options = None
+    if cmdData.input_proxy_host is not None and cmdData.input_proxy_port != 0:
+        proxy_options = http.HttpProxyOptions(
+            host_name=cmdData.input_proxy_host,
+            port=cmdData.input_proxy_port)
+
+    # Create a MQTT connection from the command line data
+    mqtt_connection = mqtt_connection_builder.mtls_from_path(
+        endpoint=cmdData.input_endpoint,
+        port=cmdData.input_port,
+        cert_filepath=cmdData.input_cert,
+        pri_key_filepath=cmdData.input_key,
+        ca_filepath=cmdData.input_ca,
+        on_connection_interrupted=on_connection_interrupted,
+        on_connection_resumed=on_connection_resumed,
+        client_id=cmdData.input_clientId,
+        clean_session=False,
+        keep_alive_secs=30,
+        http_proxy_options=proxy_options)
+
+    if not cmdData.input_is_ci:
+        print(f"Connecting to {cmdData.input_endpoint} with client ID '{cmdData.input_clientId}'...")
     else:
         print("Connecting to endpoint with client ID")
 
     connect_future = mqtt_connection.connect()
-
     # Future.result() waits until a result is available
     connect_future.result()
     print("Connected!")

samples/basic_discovery.py

@@ -3,64 +3,42 @@
 
 import time
 import json
-from concurrent.futures import Future
 from awscrt import io, http
 from awscrt.mqtt import QoS
 from awsiot.greengrass_discovery import DiscoveryClient
 from awsiot import mqtt_connection_builder
 
+from utils.command_line_utils import CommandLineUtils
+
 allowed_actions = ['both', 'publish', 'subscribe']
 
-# Parse arguments
-import utils.command_line_utils as command_line_utils
-cmdUtils = command_line_utils.CommandLineUtils("Basic Discovery - Greengrass discovery example.")
-cmdUtils.add_common_mqtt_commands()
-cmdUtils.add_common_topic_message_commands()
-cmdUtils.add_common_logging_commands()
-cmdUtils.register_command("key", "<path>", "Path to your key in PEM format.", True, str)
-cmdUtils.register_command("cert", "<path>", "Path to your client certificate in PEM format.", True, str)
-cmdUtils.remove_command("endpoint")
-cmdUtils.register_command("thing_name", "<str>", "The name assigned to your IoT Thing", required=True)
-cmdUtils.register_command(
-    "mode", "<mode>",
-    f"The operation mode (optional, default='both').\nModes:{allowed_actions}", default='both')
-cmdUtils.register_command("region", "<str>", "The region to connect through.", required=True)
-cmdUtils.register_command(
-    "max_pub_ops", "<int>",
-    "The maximum number of publish operations (optional, default='10').",
-    default=10, type=int)
-cmdUtils.register_command(
-    "print_discover_resp_only", "", "(optional, default='False').",
-    default=False, type=bool, action="store_true")
-cmdUtils.add_common_proxy_commands()
-# Needs to be called so the command utils parse the commands
-cmdUtils.get_args()
-
-tls_options = io.TlsContextOptions.create_client_with_mtls_from_path(
-    cmdUtils.get_command_required("cert"), cmdUtils.get_command_required("key"))
-if cmdUtils.get_command(cmdUtils.m_cmd_ca_file):
-    tls_options.override_default_trust_store_from_path(None, cmdUtils.get_command(cmdUtils.m_cmd_ca_file))
+# cmdData is the arguments/input from the command line placed into a single struct for
+# use in this sample. This handles all of the command line parsing, validating, etc.
+# See the Utils/CommandLineUtils for more information.
+cmdData = CommandLineUtils.parse_sample_input_basic_discovery()
+
+tls_options = io.TlsContextOptions.create_client_with_mtls_from_path(cmdData.input_cert, cmdData.input_key)
+if (cmdData.input_ca is not None):
+    tls_options.override_default_trust_store_from_path(None, cmdData.input_ca)
 tls_context = io.ClientTlsContext(tls_options)
 
 socket_options = io.SocketOptions()
 
 proxy_options = None
-if cmdUtils.get_command(cmdUtils.m_cmd_proxy_host) != None and cmdUtils.get_command(cmdUtils.m_cmd_proxy_port) != None:
-    proxy_options = http.HttpProxyOptions(
-        cmdUtils.get_command_required(cmdUtils.m_cmd_proxy_host),
-        cmdUtils.get_command_required(cmdUtils.m_cmd_proxy_port))
+if cmdData.input_proxy_host is not None and cmdData.input_proxy_port != 0:
+    proxy_options = http.HttpProxyOptions(cmdData.input_proxy_host, cmdData.input_proxy_port)
 
 print('Performing greengrass discovery...')
 discovery_client = DiscoveryClient(
     io.ClientBootstrap.get_or_create_static_default(),
     socket_options,
     tls_context,
-    cmdUtils.get_command_required("region"), None, proxy_options)
-resp_future = discovery_client.discover(cmdUtils.get_command_required("thing_name"))
+    cmdData.input_signing_region, None, proxy_options)
+resp_future = discovery_client.discover(cmdData.input_thing_name)
 discover_response = resp_future.result()
 
 print(discover_response)
-if cmdUtils.get_command("print_discover_resp_only"):
+if (cmdData.input_print_discovery_resp_only):
     exit(0)
 
 
@@ -78,16 +56,17 @@ def try_iot_endpoints():
         for gg_core in gg_group.cores:
             for connectivity_info in gg_core.connectivity:
                 try:
-                    print (f"Trying core {gg_core.thing_arn} at host {connectivity_info.host_address} port {connectivity_info.port}")
+                    print(
+                        f"Trying core {gg_core.thing_arn} at host {connectivity_info.host_address} port {connectivity_info.port}")
                     mqtt_connection = mqtt_connection_builder.mtls_from_path(
                         endpoint=connectivity_info.host_address,
                         port=connectivity_info.port,
-                        cert_filepath=cmdUtils.get_command_required("cert"),
-                        pri_key_filepath=cmdUtils.get_command_required("key"),
+                        cert_filepath=cmdData.input_cert,
+                        pri_key_filepath=cmdData.input_key,
                         ca_bytes=gg_group.certificate_authorities[0].encode('utf-8'),
                         on_connection_interrupted=on_connection_interupted,
                         on_connection_resumed=on_connection_resumed,
-                        client_id=cmdUtils.get_command_required("thing_name"),
+                        client_id=cmdData.input_clientId,
                         clean_session=False,
                         keep_alive_secs=30)
 
@@ -102,27 +81,26 @@ def try_iot_endpoints():
 
     exit('All connection attempts failed')
 
-mqtt_connection = try_iot_endpoints()
 
-if cmdUtils.get_command("mode") == 'both' or cmdUtils.get_command("mode") == 'subscribe':
+mqtt_connection = try_iot_endpoints()
 
+if cmdData.input_mode == 'both' or cmdData.input_mode == 'subscribe':
     def on_publish(topic, payload, dup, qos, retain, **kwargs):
         print('Publish received on topic {}'.format(topic))
         print(payload)
-
-    subscribe_future, _ = mqtt_connection.subscribe(cmdUtils.get_command("topic"), QoS.AT_MOST_ONCE, on_publish)
+    subscribe_future, _ = mqtt_connection.subscribe(cmdData.input_topic, QoS.AT_MOST_ONCE, on_publish)
     subscribe_result = subscribe_future.result()
 
 loop_count = 0
-while loop_count < cmdUtils.get_command("max_pub_ops"):
-    if cmdUtils.get_command("mode") == 'both' or cmdUtils.get_command("mode") == 'publish':
+while loop_count < cmdData.input_max_pub_ops:
+    if cmdData.input_mode == 'both' or cmdData.input_mode == 'publish':
         message = {}
-        message['message'] = cmdUtils.get_command("message")
+        message['message'] = cmdData.input_message
         message['sequence'] = loop_count
         messageJson = json.dumps(message)
-        pub_future, _ = mqtt_connection.publish(cmdUtils.get_command("topic"), messageJson, QoS.AT_MOST_ONCE)
+        pub_future, _ = mqtt_connection.publish(cmdData.input_topic, messageJson, QoS.AT_MOST_ONCE)
         pub_future.result()
-        print('Published topic {}: {}\n'.format(cmdUtils.get_command("topic"), messageJson))
+        print('Published topic {}: {}\n'.format(cmdData.input_topic, messageJson))
 
         loop_count += 1
     time.sleep(1)