Refactor of command line argument parsing to single file (#278)

Simplifies command line parsing and ensures naming consistency between
samples and the other AWS IoT SDKs.

* Simplified command line parsing
* Fixed tests not working correctly after simplified command line parsing
* Added action support to command line parser
* Added additional command groups

Author: TwistedTwigleg

codebuild/samples/pubsub-linux.sh

@@ -12,6 +12,6 @@ echo "Mqtt Direct test"
 python3 pubsub.py --endpoint $ENDPOINT --key /tmp/privatekey.pem --cert /tmp/certificate.pem
 
 echo "Websocket test"
-python3 pubsub.py --endpoint $ENDPOINT --use-websocket --signing-region us-east-1
+python3 pubsub.py --endpoint $ENDPOINT --use_websocket --signing_region us-east-1
 
 popd

samples/README.md

@@ -23,7 +23,7 @@ Source: `samples/pubsub.py`
 
 Run the sample like this:
 ``` sh
-python3 pubsub.py --endpoint <endpoint> --root-ca <file> --cert <file> --key <file>
+python3 pubsub.py --endpoint <endpoint> --ca_file <file> --cert <file> --key <file>
 ```
 
 Your Thing's
@@ -120,7 +120,7 @@ To run this sample using [SoftHSM2](https://www.opendnssec.org/softhsm/) as the
 
 5)  Now you can run the sample:
     ```sh
-    python3 pkcs11_pubsub.py --endpoint <xxxx-ats.iot.xxxx.amazonaws.com> --root-ca <AmazonRootCA1.pem> --cert <certificate.pem.crt> --pkcs11-lib <libsofthsm2.so> --pin <user-pin> --token-label <token-label> --key-label <key-label>
+    python3 pkcs11_pubsub.py --endpoint <xxxx-ats.iot.xxxx.amazonaws.com> --ca_file <AmazonRootCA1.pem> --cert <certificate.pem.crt> --pkcs11_lib <libsofthsm2.so> --pin <user-pin> --token_label <token-label> --key_label <key-label>
 
 
 ## Windows Certificate PubSub
@@ -210,7 +210,7 @@ Source: `samples/shadow.py`
 
 Run the sample like this:
 ``` sh
-python3 shadow.py --endpoint <endpoint> --root-ca <file> --cert <file> --key <file> --thing-name <name>
+python3 shadow.py --endpoint <endpoint> --ca_file <file> --cert <file> --key <file> --thing-name <name>
 ```
 
 Your Thing's
@@ -296,7 +296,7 @@ Source: `samples/jobs.py`
 
 Run the sample like this:
 ``` sh
-python3 jobs.py --endpoint <endpoint> --root-ca <file> --cert <file> --key <file> --thing-name <name>
+python3 jobs.py --endpoint <endpoint> --ca_file <file> --cert <file> --key <file> --thing_name <name>
 ```
 
 Your Thing's
@@ -368,12 +368,12 @@ Source: `samples/fleetprovisioning.py`
 
 Run the sample using createKeysAndCertificate:
 ``` sh
-python3 fleetprovisioning.py --endpoint <endpoint> --root-ca <file> --cert <file> --key <file> --templateName <name> --templateParameters <parameters>
+python3 fleetprovisioning.py --endpoint <endpoint> --ca_file <file> --cert <file> --key <file> --template_name <name> --template_parameters <parameters>
 ```
 
 Run the sample using createCertificateFromCsr:
 ``` sh
-python3 fleetprovisioning.py --endpoint <endpoint> --root-ca <file> --cert <file> --key <file> --templateName <name> --templateParameters <parameters> --csr <csr file>
+python3 fleetprovisioning.py --endpoint <endpoint> --ca_file <file> --cert <file> --key <file> --template_name <name> --template_parameters <parameters> --csr <csr file>
 ```
 
 Your Thing's
@@ -489,11 +489,11 @@ and `--key` appropriately:
 ``` sh
 python3 fleetprovisioning.py \
         --endpoint [your endpoint]-ats.iot.[region].amazonaws.com \
-        --root-ca [pathToRootCA] \
+        --ca_file [pathToRootCA] \
         --cert /tmp/provision.cert.pem \
         --key /tmp/provision.private.key \
-        --templateName [TemplateName] \
-        --templateParameters "{\"SerialNumber\":\"1\",\"DeviceLocation\":\"Seattle\"}"
+        --template_name [TemplateName] \
+        --template_parameters "{\"SerialNumber\":\"1\",\"DeviceLocation\":\"Seattle\"}"
 ```
 
 Notice that we provided substitution values for the two parameters in the template body, `DeviceLocation` and `SerialNumber`.
@@ -529,11 +529,11 @@ using a permanent certificate set, replace the paths specified in the `--cert` a
 ``` sh
 python3 fleetprovisioning.py \
         --endpoint [your endpoint]-ats.iot.[region].amazonaws.com \
-        --root-ca [pathToRootCA] \
+        --ca_file [pathToRootCA] \
         --cert /tmp/provision.cert.pem \
         --key /tmp/provision.private.key \
-        --templateName [TemplateName] \
-        --templateParameters "{\"SerialNumber\":\"1\",\"DeviceLocation\":\"Seattle\"}" \
+        --template_name [TemplateName] \
+        --template_parameters "{\"SerialNumber\":\"1\",\"DeviceLocation\":\"Seattle\"}" \
         --csr /tmp/deviceCert.csr
 ```
 

samples/basic_discovery.py

@@ -14,29 +14,25 @@
 
 allowed_actions = ['both', 'publish', 'subscribe']
 
-parser = argparse.ArgumentParser()
-parser.add_argument('-r', '--root-ca', action='store', dest='root_ca_path', help='Root CA file path')
-parser.add_argument('-c', '--cert', action='store', required=True, dest='certificate_path', help='Certificate file path')
-parser.add_argument('-k', '--key', action='store', required=True, dest='private_key_path', help='Private key file path')
-parser.add_argument('-n', '--thing-name', action='store', required=True, dest='thing_name', help='Targeted thing name')
-parser.add_argument('-t', '--topic', action='store', dest='topic', default='test/topic', help='Targeted topic')
-parser.add_argument('-m', '--mode', action='store', dest='mode', default='both',
-                    help='Operation modes: %s'%str(allowed_actions))
-parser.add_argument('-M', '--message', action='store', dest='message', default='Hello World!',
-                    help='Message to publish')
-parser.add_argument('--region', action='store', dest='region', default='us-east-1')
-parser.add_argument('--max-pub-ops', action='store', dest='max_pub_ops', default=10)
-parser.add_argument('--print-discover-resp-only', action='store_true', dest='print_discover_resp_only', default=False)
-parser.add_argument('-v', '--verbosity', choices=[x.name for x in LogLevel], default=LogLevel.NoLogs.name,
-                    help='Logging level')
-
-args = parser.parse_args()
+# Parse arguments
+import command_line_utils;
+cmdUtils = command_line_utils.CommandLineUtils("Basic Discovery - Greengrass discovery example.")
+cmdUtils.add_common_mqtt_commands()
+cmdUtils.add_common_topic_message_commands()
+cmdUtils.add_common_logging_commands()
+cmdUtils.remove_command("endpoint")
+cmdUtils.register_command("thing_name", "<str>", "The name assigned to your IoT Thing", required=True)
+cmdUtils.register_command("mode", "<mode>", "The operation mode (optional, default='both').\nModes:%s"%str(allowed_actions), default='both')
+cmdUtils.register_command("region", "<str>", "The region to connect through (optional, default='us-east-1').", default="us-east-1")
+cmdUtils.register_command("max_pub_ops", "<int>", "The maximum number of publish operations (optional, default='10').", default=10, type=int)
+cmdUtils.register_command("print_discover_resp_only", "", "(optional, default='False').", default=False, type=bool, action="store_true")
+args = cmdUtils.get_args()
 
 io.init_logging(getattr(LogLevel, args.verbosity), 'stderr')
 
-tls_options = io.TlsContextOptions.create_client_with_mtls_from_path(args.certificate_path, args.private_key_path)
-if args.root_ca_path:
-    tls_options.override_default_trust_store_from_path(None, args.root_ca_path)
+tls_options = io.TlsContextOptions.create_client_with_mtls_from_path(args.cert, args.key)
+if args.ca_file:
+    tls_options.override_default_trust_store_from_path(None, args.ca_file)
 tls_context = io.ClientTlsContext(tls_options)
 
 socket_options = io.SocketOptions()
@@ -69,8 +65,8 @@ def try_iot_endpoints():
                     mqtt_connection = mqtt_connection_builder.mtls_from_path(
                         endpoint=connectivity_info.host_address,
                         port=connectivity_info.port,
-                        cert_filepath=args.certificate_path,
-                        pri_key_filepath=args.private_key_path,
+                        cert_filepath=args.cert,
+                        pri_key_filepath=args.key,
                         ca_bytes=gg_group.certificate_authorities[0].encode('utf-8'),
                         on_connection_interrupted=on_connection_interupted,
                         on_connection_resumed=on_connection_resumed,

samples/command_line_utils.py

@@ -0,0 +1,75 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0.
+
+import argparse
+from awscrt import io
+
+class CommandLineUtils:
+    def __init__(self, description) -> None:
+        self.parser = argparse.ArgumentParser(description="Send and receive messages through and MQTT connection.")
+        self.commands = {}
+
+    def register_command(self, command_name, example_input, help_output, required=False, type=None, default=None, choices=None, action=None):
+        self.commands[command_name] = {
+            "name":command_name,
+            "example_input":example_input,
+            "help_output":help_output,
+            "required": required,
+            "type": type,
+            "default": default,
+            "choices": choices,
+            "action": action
+        }
+
+    def remove_command(self, command_name):
+        if command_name in self.commands.keys():
+            self.commands.pop(command_name)
+
+    def get_args(self):
+        # add all the commands
+        for command in self.commands.values():
+            if not command["action"] is None:
+                self.parser.add_argument("--" + command["name"], action=command["action"], help=command["help_output"],
+                    required=command["required"], default=command["default"])
+            else:
+                self.parser.add_argument("--" + command["name"], metavar=command["example_input"], help=command["help_output"],
+                    required=command["required"], type=command["type"], default=command["default"], choices=command["choices"])
+
+        return self.parser.parse_args()
+
+    def update_command(self, command_name, new_example_input=None, new_help_output=None, new_required=None, new_type=None, new_default=None, new_action=None):
+        if command_name in self.commands.keys():
+            if new_example_input:
+                self.commands[command_name]["example_input"] = new_example_input
+            if new_help_output:
+                self.commands[command_name]["help_output"] = new_help_output
+            if new_required:
+                self.commands[command_name]["required"] = new_required
+            if new_type:
+                self.commands[command_name]["type"] = new_type
+            if new_default:
+                self.commands[command_name]["default"] = new_default
+            if new_action:
+                self.commands[command_name]["action"] = new_action
+
+    def add_common_mqtt_commands(self):
+        self.register_command("endpoint", "<str>", "The endpoint of the mqtt server not including a port.", True, str)
+        self.register_command("key", "<path>", "Path to your key in PEM format.", False, str)
+        self.register_command("cert", "<path>", "Path to your client certificate in PEM format.", False, str)
+        self.register_command("ca_file", "<path>", "Path to AmazonRootCA1.pem (optional, system trust store used by default)", False, str)
+
+    def add_common_proxy_commands(self):
+        self.register_command("proxy_host", "<str>", "Host name of the proxy server to connect through (optional)", False, str)
+        self.register_command("proxy_port", "<int>", "Port of the http proxy to use (optional, default='8080')", type=int, default=8080)
+
+    def add_common_topic_message_commands(self):
+        self.register_command("topic", "<str>", "Topic to publish, subscribe to (optional, default='test/topic').", default="test/topic")
+        self.register_command("message", "<str>", "The message to send in the payload (optional, default='Hello World!').", default="Hello World!")
+
+    def add_common_websocket_commands(self):
+        self.register_command("use_websocket", "", "If specified, uses a websocket over https (optional).", default=False, action="store_true")
+        self.register_command("signing_region", "<str>",
+            "Used for websocket signer. It should only be specified if websockets are used (optional, default='us-east-1')", default="us-east-1")
+
+    def add_common_logging_commands(self):
+        self.register_command("verbosity", "<Log Level>", "Logging level.", default=io.LogLevel.NoLogs.name, choices=[x.name for x in io.LogLevel])

samples/fleetprovisioning.py

@@ -26,31 +26,22 @@
 # On startup, the script subscribes to topics based on the request type of either CSR or Keys
 # publishes the request to corresponding topic and calls RegisterThing.
 
-parser = argparse.ArgumentParser(description="Fleet Provisioning sample script.")
-parser.add_argument('--endpoint', required=True, help="Your AWS IoT custom endpoint, not including a port. " +
-                                                      "Ex: \"w6zbse3vjd5b4p-ats.iot.us-west-2.amazonaws.com\"")
-parser.add_argument('--cert', help="File path to your client certificate, in PEM format")
-parser.add_argument('--key', help="File path to your private key file, in PEM format")
-parser.add_argument('--root-ca', help="File path to root certificate authority, in PEM format. " +
-                                      "Necessary if MQTT server uses a certificate that's not already in " +
-                                      "your trust store")
-parser.add_argument('--client-id', default="test-" + str(uuid4()), help="Client ID for MQTT connection.")
-parser.add_argument('--use-websocket', default=False, action='store_true',
-                    help="To use a websocket instead of raw mqtt. If you " +
-                         "specify this option you must specify a region for signing.")
-parser.add_argument('--signing-region', default='us-east-1', help="If you specify --use-web-socket, this " +
-                                                                  "is the region that will be used for computing the Sigv4 signature")
-parser.add_argument('--proxy-host', help="Hostname of proxy to connect to.")
-parser.add_argument('--proxy-port', type=int, default=8080, help="Port of proxy to connect to.")
-parser.add_argument('--verbosity', choices=[x.name for x in io.LogLevel], default=io.LogLevel.NoLogs.name,
-                    help='Logging level')
-parser.add_argument("--csr", help="File path to your client CSR in PEM format")
-parser.add_argument("--templateName", help="Template name")
-parser.add_argument("--templateParameters", help="Values for Template Parameters")
+# Parse arguments
+import command_line_utils;
+cmdUtils = command_line_utils.CommandLineUtils("Fleet Provisioning - Provision device using either the keys or CSR.")
+cmdUtils.add_common_mqtt_commands()
+cmdUtils.add_common_websocket_commands()
+cmdUtils.add_common_proxy_commands()
+cmdUtils.add_common_logging_commands()
+cmdUtils.register_command("client_id", "<str>", "Client ID to use for MQTT connection (optional, default='test-*').", default="test-" + str(uuid4()))
+cmdUtils.register_command("csr", "<path>", "Path to CSR in Pem format (optional).")
+cmdUtils.register_command("template_name", "<str>", "The name of your provisioning template.")
+cmdUtils.register_command("template_parameters", "<json>", "Template parameters json.")
+cmdUtils.update_command("cert", new_help_output="Path to your certificate in PEM format. If this is not set you must specify use_websocket")
+args = cmdUtils.get_args()
 
 # Using globals to simplify sample code
 is_sample_done = threading.Event()
-args = parser.parse_args()
 
 io.init_logging(getattr(io.LogLevel, args.verbosity), 'stderr')
 mqtt_connection = None
@@ -240,7 +231,7 @@ def waitForRegisterThingResponse():
             http_proxy_options=proxy_options,
             on_connection_interrupted=on_connection_interrupted,
             on_connection_resumed=on_connection_resumed,
-            ca_filepath=args.root_ca,
+            ca_filepath=args.ca_file,
             client_id=args.client_id,
             clean_session=False,
             keep_alive_secs=30)
@@ -250,7 +241,7 @@ def waitForRegisterThingResponse():
             endpoint=args.endpoint,
             cert_filepath=args.cert,
             pri_key_filepath=args.key,
-            ca_filepath=args.root_ca,
+            ca_filepath=args.ca_file,
             client_id=args.client_id,
             on_connection_interrupted=on_connection_interrupted,
             on_connection_resumed=on_connection_resumed,
@@ -321,7 +312,7 @@ def waitForRegisterThingResponse():
             createcertificatefromcsr_subscribed_rejected_future.result()
 
 
-        registerthing_subscription_request = iotidentity.RegisterThingSubscriptionRequest(template_name=args.templateName)
+        registerthing_subscription_request = iotidentity.RegisterThingSubscriptionRequest(template_name=args.template_name)
 
         print("Subscribing to RegisterThing Accepted topic...")
         registerthing_subscribed_accepted_future, _ = identity_client.subscribe_to_register_thing_accepted(
@@ -352,9 +343,9 @@ def waitForRegisterThingResponse():
                 raise Exception('CreateKeysAndCertificate API did not succeed')
 
             registerThingRequest = iotidentity.RegisterThingRequest(
-                template_name=args.templateName,
+                template_name=args.template_name,
                 certificate_ownership_token=createKeysAndCertificateResponse.certificate_ownership_token,
-                parameters=json.loads(args.templateParameters))
+                parameters=json.loads(args.template_parameters))
         else:
             print("Publishing to CreateCertificateFromCsr...")
             csrPath = open(args.csr, 'r').read()
@@ -369,9 +360,9 @@ def waitForRegisterThingResponse():
                 raise Exception('CreateCertificateFromCsr API did not succeed')
 
             registerThingRequest = iotidentity.RegisterThingRequest(
-                template_name=args.templateName,
+                template_name=args.template_name,
                 certificate_ownership_token=createCertificateFromCsrResponse.certificate_ownership_token,
-                parameters=json.loads(args.templateParameters))
+                parameters=json.loads(args.template_parameters))
 
         print("Publishing to RegisterThing topic...")
         registerthing_publish_future = identity_client.publish_register_thing(registerThingRequest, mqtt.QoS.AT_LEAST_ONCE)