Added custom authorizer sample (#315)

Adds custom authorizer support and sample to the Python V2 SDK

Commit log:
* Added custom authorizer sample
* Fixed documentation generation error and fixed custom authorizer password not being set correctly in CI
* Added custom authorizer sample to README
* Code review changes: fixed naming and cleaned up code

Author: TwistedTwigleg

awsiot/mqtt_connection_builder.py

@@ -127,26 +127,38 @@ def _get(kwargs, name, default=None):
 _metrics_str = None
 
 
-def _get_metrics_str():
+def _get_metrics_str(current_username=""):
     global _metrics_str
+
+    username_has_query = False
+    if not current_username.find("?") is -1:
+        username_has_query = True
+
     if _metrics_str is None:
         try:
             import pkg_resources
             try:
                 version = pkg_resources.get_distribution("awsiotsdk").version
-                _metrics_str = "?SDK=PythonV2&Version={}".format(version)
+                _metrics_str = "SDK=PythonV2&Version={}".format(version)
             except pkg_resources.DistributionNotFound:
-                _metrics_str = "?SDK=PythonV2&Version=dev"
+                _metrics_str = "SDK=PythonV2&Version=dev"
         except BaseException:
             _metrics_str = ""
 
-    return _metrics_str
+    if not _metrics_str == "":
+        if username_has_query:
+            return "&" + _metrics_str
+        else:
+            return "?" + _metrics_str
+    else:
+        return ""
 
 
 def _builder(
         tls_ctx_options,
         use_websockets=False,
         websocket_handshake_transform=None,
+        use_custom_authorizer=False,
         **kwargs):
 
     ca_bytes = _get(kwargs, 'ca_bytes')
@@ -165,7 +177,7 @@ def _builder(
         else:
             port = 8883
 
-    if port == 443 and awscrt.io.is_alpn_available():
+    if port == 443 and awscrt.io.is_alpn_available() and use_custom_authorizer is False:
         tls_ctx_options.alpn_list = ['http/1.1'] if use_websockets else ['x-amzn-mqtt-ca']
 
     socket_options = awscrt.io.SocketOptions()
@@ -185,7 +197,7 @@ def _builder(
 
     username = _get(kwargs, 'username', '')
     if _get(kwargs, 'enable_metrics_collection', True):
-        username += _get_metrics_str()
+        username += _get_metrics_str(username)
 
     client_bootstrap = _get(kwargs, 'client_bootstrap')
     if client_bootstrap is None:
@@ -414,3 +426,92 @@ def websockets_with_custom_handshake(
                     websocket_handshake_transform=websocket_handshake_transform,
                     websocket_proxy_options=websocket_proxy_options,
                     **kwargs)
+
+def _add_to_username_parameter(input_string, parameter_value, parameter_pretext):
+    """
+    Helper function to add parameters to the username in the direct_with_custom_authorizer function
+    """
+    return_string = input_string
+
+    if not return_string.find("?") is -1:
+        return_string += "&"
+    else:
+        return_string += "?"
+
+    if not parameter_value.find(parameter_pretext) is -1:
+        return return_string + parameter_value
+    else:
+        return return_string + parameter_pretext + parameter_value
+
+def direct_with_custom_authorizer(
+        auth_username=None,
+        auth_authorizer_name=None,
+        auth_authorizer_signature=None,
+        auth_password=None,
+        **kwargs) -> awscrt.mqtt.Connection:
+    """
+    This builder creates an :class:`awscrt.mqtt.Connection`, configured for an MQTT connection using a custom
+    authorizer. This function will set the username, port, and TLS options.
+
+    This function takes all :mod:`common arguments<awsiot.mqtt_connection_builder>`
+    described at the top of this doc, as well as...
+
+    Keyword Args:
+        auth_username (`str`): The username to use with the custom authorizer.
+            If provided, the username given will be passed when connecting to the custom authorizer.
+            If not provided, it will check to see if a username has already been set (via username="example")
+            and will use that instead.
+            If no username has been set then no username will be sent with the MQTT connection.
+
+        auth_authorizer_name (`str`):  The name of the custom authorizer.
+            If not provided, then "x-amz-customauthorizer-name" will not be added with the MQTT connection.
+
+        auth_authorizer_signature (`str`):  The signature of the custom authorizer.
+            If not provided, then "x-amz-customauthorizer-name" will not be added with the MQTT connection.
+
+        auth_password (`str`):  The password to use with the custom authorizer.
+            If not provided, then no passord will be set.
+    """
+
+    _check_required_kwargs(**kwargs)
+    username_string = ""
+
+    if auth_username is None:
+        if not _get(kwargs, "username") is None:
+            username_string += _get(kwargs, "username")
+    else:
+        username_string += auth_username
+
+    if not auth_authorizer_name is None:
+        username_string = _add_to_username_parameter(
+            username_string, auth_authorizer_name, "x-amz-customauthorizer-name=")
+    if not auth_authorizer_signature is None:
+        username_string = _add_to_username_parameter(
+            username_string, auth_authorizer_signature, "x-amz-customauthorizer-signature=")
+
+    kwargs["username"] = username_string
+    kwargs["password"] = auth_password
+    kwargs["port"] = 443
+
+    tls_ctx_options = awscrt.io.TlsContextOptions()
+    tls_ctx_options.alpn_list = ["mqtt"]
+
+    return _builder(tls_ctx_options=tls_ctx_options,
+                    use_websockets=False,
+                    use_custom_authorizer=True,
+                    **kwargs)
+
+
+def new_default_builder(**kwargs) -> awscrt.mqtt.Connection:
+    """
+    This builder creates an :class:`awscrt.mqtt.Connection`, without any configuration besides the default TLS context options.
+
+    This requires setting the connection details manually by passing all the necessary data
+    in :mod:`common arguments<awsiot.mqtt_connection_builder>` to make a connection
+    """
+    _check_required_kwargs(kwargs)
+    tls_ctx_options = awscrt.io.TlsContextOptions()
+
+    return _builder(tls_ctx_options=tls_ctx_options,
+                    use_websockets=False,
+                    kwargs=kwargs)

codebuild/samples/connect-auth-linux.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+set -e
+
+env
+
+pushd $CODEBUILD_SRC_DIR/samples/
+
+ENDPOINT=$(aws secretsmanager get-secret-value --secret-id "unit-test/endpoint" --query "SecretString" | cut -f2 -d":" | sed -e 's/[\\\"\}]//g')
+AUTH_NAME=$(aws secretsmanager get-secret-value --secret-id "unit-test/authorizer-name" --query "SecretString" | cut -f2 -d":" | sed -e 's/[\\\"\}]//g')
+AUTH_PASSWORD=$(aws secretsmanager get-secret-value --secret-id "unit-test/authorizer-password" --query "SecretString" | cut -f2 -d":" | sed -e 's/[\\\"\}]//g')
+
+echo "Custom authorizer connect test"
+python3 custom_authorizer_connect.py --endpoint $ENDPOINT --custom_auth_authorizer_name $AUTH_NAME --custom_auth_password $AUTH_PASSWORD
+
+popd

codebuild/samples/linux-smoke-tests.yml

@@ -12,6 +12,7 @@ phases:
       - $CODEBUILD_SRC_DIR/codebuild/samples/connect-linux.sh
       - $CODEBUILD_SRC_DIR/codebuild/samples/pkcs11-connect-linux.sh
       - $CODEBUILD_SRC_DIR/codebuild/samples/pubsub-linux.sh
+      - $CODEBUILD_SRC_DIR/codebuild/samples/connect-auth-linux.sh
   post_build:
     commands:
       - echo Build completed on `date`

samples/README.md

@@ -5,6 +5,7 @@
 * [Websocket Connect](#websocket-connect)
 * [PKCS#11 Connect](#pkcs11-connect)
 * [Windows Certificate Connect](#windows-certificate-connect)
+* [Custom Authorizer Connect](#custom-authorizer-connect)
 * [Shadow](#shadow)
 * [Jobs](#jobs)
 * [Fleet Provisioning](#fleet-provisioning)
@@ -324,6 +325,39 @@ To run this sample with a basic certificate from AWS IoT Core:
     python3 windows_cert_connect.py --endpoint <endpoint> --ca_file <path to root CA> --cert <path to certificate>
     ```
 
+## Custom Authorizer Connect
+
+This sample makes an MQTT connection and connects through a [Custom Authorizer](https://docs.aws.amazon.com/iot/latest/developerguide/custom-authentication.html). On startup, the device connects to the server and then disconnects. This sample is for reference on connecting using a custom authorizer.
+
+Your Thing's [Policy](https://docs.aws.amazon.com/iot/latest/developerguide/iot-policies.html) must provide privileges for this sample to connect.
+
+<details>
+<summary>(see sample policy)</summary>
+<pre>
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Action": [
+        "iot:Connect"
+      ],
+      "Resource": [
+        "arn:aws:iot:<b>region</b>:<b>account</b>:client/test-*"
+      ]
+    }
+  ]
+}
+</pre>
+</details>
+
+Run the sample like this:
+``` sh
+python3 custom_authorizer_connect.py --endpoint <endpoint> --ca_file <path to root CA> --custom_auth_authorizer_name <authorizer name>
+```
+
+You will need to setup your Custom Authorizer so that the lambda function returns a policy document. See [this page on the documentation](https://docs.aws.amazon.com/iot/latest/developerguide/config-custom-auth.html) for more details and example return result.
+
 ## Shadow
 
 This sample uses the AWS IoT

samples/command_line_utils.py

@@ -77,6 +77,12 @@ def add_common_topic_message_commands(self):
     def add_common_logging_commands(self):
         self.register_command(self.m_cmd_verbosity, "<Log Level>", "Logging level.", default=io.LogLevel.NoLogs.name, choices=[x.name for x in io.LogLevel])
 
+    def add_common_custom_authorizer_commands(self):
+        self.register_command(self.m_cmd_custom_auth_username, "<str>", "The name to send when connecting through the custom authorizer (optional)")
+        self.register_command(self.m_cmd_custom_auth_authorizer_name, "<str>", "The name of the custom authorizer to connect to (optional but required for everything but custom domains)")
+        self.register_command(self.m_cmd_custom_auth_username, "<str>", "The signature to send when connecting through a custom authorizer (optional)")
+        self.register_command(self.m_cmd_custom_auth_password, "<str>", "The password to send when connecting through a custom authorizer (optional)")
+
     """
     Returns the command if it exists and has been passed to the console, otherwise it will print the help for the sample and exit the application.
     """
@@ -191,3 +197,7 @@ def get_proxy_options_for_mqtt_connection(self):
     m_cmd_message = "message"
     m_cmd_topic = "topic"
     m_cmd_verbosity = "verbosity"
+    m_cmd_custom_auth_username = "custom_auth_username"
+    m_cmd_custom_auth_authorizer_name = "custom_auth_authorizer_name"
+    m_cmd_custom_auth_authorizer_signature = "custom_auth_authorizer_signature"
+    m_cmd_custom_auth_password = "custom_auth_password"

samples/custom_authorizer_connect.py

@@ -0,0 +1,63 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0.
+
+from awsiot import mqtt_connection_builder
+from uuid import uuid4
+
+# This sample is similar to `samples/basic_connect.py` but it connects
+# through a custom authorizer rather than using a key and certificate.
+
+# Parse arguments
+import command_line_utils
+cmdUtils = command_line_utils.CommandLineUtils(
+    "Custom Authorizer Connect - Make a MQTT connection using a custom authorizer.")
+cmdUtils.add_common_mqtt_commands()
+cmdUtils.add_common_logging_commands()
+cmdUtils.add_common_custom_authorizer_commands()
+cmdUtils.register_command("client_id", "<str>",
+                          "Client ID to use for MQTT connection (optional, default='test-*').",
+                          default="test-" + str(uuid4()))
+# Needs to be called so the command utils parse the commands
+cmdUtils.get_args()
+
+
+def on_connection_interrupted(connection, error, **kwargs):
+    # Callback when connection is accidentally lost.
+    print("Connection interrupted. error: {}".format(error))
+
+
+def on_connection_resumed(connection, return_code, session_present, **kwargs):
+    # Callback when an interrupted connection is re-established.
+    print("Connection resumed. return_code: {} session_present: {}".format(return_code, session_present))
+
+
+if __name__ == '__main__':
+
+    # Create MQTT connection with a custom authorizer
+    mqtt_connection = mqtt_connection_builder.direct_with_custom_authorizer(
+        endpoint=cmdUtils.get_command_required(cmdUtils.m_cmd_endpoint),
+        ca_filepath=cmdUtils.get_command(cmdUtils.m_cmd_ca_file),
+        auth_username=cmdUtils.get_command(cmdUtils.m_cmd_custom_auth_username),
+        auth_authorizer_name=cmdUtils.get_command(cmdUtils.m_cmd_custom_auth_authorizer_name),
+        auth_authorizer_signature=cmdUtils.get_command(cmdUtils.m_cmd_custom_auth_authorizer_signature),
+        auth_password=cmdUtils.get_command(cmdUtils.m_cmd_custom_auth_password),
+        on_connection_interrupted=on_connection_interrupted,
+        on_connection_resumed=on_connection_resumed,
+        client_id=cmdUtils.get_command("client_id"),
+        clean_session=False,
+        keep_alive_secs=30)
+
+    print("Connecting to {} with client ID '{}'...".format(
+        cmdUtils.get_command(cmdUtils.m_cmd_endpoint), cmdUtils.get_command("client_id")))
+
+    connect_future = mqtt_connection.connect()
+
+    # Future.result() waits until a result is available
+    connect_future.result()
+    print("Connected!")
+
+    # Disconnect
+    print("Disconnecting...")
+    disconnect_future = mqtt_connection.disconnect()
+    disconnect_future.result()
+    print("Disconnected!")

samples/windows_cert_connect.py

@@ -17,17 +17,9 @@
 cmdUtils = command_line_utils.CommandLineUtils("Windows Cert Connect - Make a MQTT connection using Windows Store Certificates.")
 cmdUtils.add_common_mqtt_commands()
 cmdUtils.add_common_logging_commands()
-cmdUtils.register_command("port", "<int>",
-                          "Connection port for direct connection. " +
-                          "AWS IoT supports 433 and 8883 (optional, default=8883).",
-                          False, int)
 cmdUtils.register_command("client_id", "<str>",
                           "Client ID to use for MQTT connection (optional, default='test-*').",
                           default="test-" + str(uuid4()))
-cmdUtils.register_command("cert", "<path>",
-                          "Path to certificate in Windows certificate store. " +
-                          "e.g. \"CurrentUser\\MY\\6ac133ac58f0a88b83e9c794eba156a98da39b4c\"",
-                          True, str)
 # Needs to be called so the command utils parse the commands
 cmdUtils.get_args()