diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2fd319bb..8de9f99e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,6 +23,7 @@ env: CI_SAMPLES_FOLDER: "./aws-iot-device-sdk-python-v2/samples" CI_IOT_CONTAINERS_ROLE: ${{ secrets.AWS_CI_IOT_CONTAINERS }} CI_PUBSUB_ROLE: ${{ secrets.AWS_CI_PUBSUB_ROLE }} + CI_COGNITO_ROLE: ${{ secrets.AWS_CI_COGNITO_ROLE }} CI_CUSTOM_AUTHORIZER_ROLE: ${{ secrets.AWS_CI_CUSTOM_AUTHORIZER_ROLE }} CI_SHADOW_ROLE: ${{ secrets.AWS_CI_SHADOW_ROLE }} CI_JOBS_ROLE: ${{ secrets.AWS_CI_JOBS_ROLE }} @@ -244,6 +245,14 @@ jobs: export SOFTHSM2_CONF=/tmp/softhsm2.conf echo "directories.tokendir = /tmp/tokens" > /tmp/softhsm2.conf python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --file ${{ env.CI_SAMPLES_CFG_FOLDER }}/ci_run_pkcs11_connect_cfg.json + - name: configure AWS credentials (Cognito) + uses: aws-actions/configure-aws-credentials@v1 + with: + role-to-assume: ${{ env.CI_COGNITO_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} + - name: run Cognito Connect sample + run: | + python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --file ${{ env.CI_SAMPLES_CFG_FOLDER }}/ci_run_cognito_connect_cfg.json - name: configure AWS credentials (MQTT5 samples) uses: aws-actions/configure-aws-credentials@v1 with: diff --git a/.github/workflows/ci_run_cognito_connect_cfg.json b/.github/workflows/ci_run_cognito_connect_cfg.json new file mode 100644 index 00000000..1f6b6cd1 --- /dev/null +++ b/.github/workflows/ci_run_cognito_connect_cfg.json @@ -0,0 +1,20 @@ +{ + "language": "Python", + "sample_file": "./aws-iot-device-sdk-python-v2/samples/cognito_connect.py", + "sample_region": "us-east-1", + "sample_main_class": "", + "arguments": [ + { + "name": "--endpoint", + "secret": "ci/endpoint" + }, + { + "name": "--signing_region", + "data": "us-east-1" + }, + { + "name": "--cognito_identity", + "secret": "ci/Cognito/identity_id" + } + ] +} diff --git a/documents/MQTT5_Userguide.md b/documents/MQTT5_Userguide.md index b984f6c0..48dc8f51 100644 --- a/documents/MQTT5_Userguide.md +++ b/documents/MQTT5_Userguide.md @@ -14,6 +14,7 @@ * [MQTT over Websockets with Sigv4 authentication](#mqtt-over-websockets-with-sigv4-authentication) * [Direct MQTT with Custom Authentication](#direct-mqtt-with-custom-authentication) * [Direct MQTT with PKCS11 Method](#direct-mqtt-with-pkcs11-method) + * [MQTT over Websockets with Cognito authentication](#mqtt-over-websockets-with-cognito-authentication) * [HTTP Proxy](#http-proxy) * [Client Lifecycle Management](#client-lifecycle-management) * [Lifecycle Events](#lifecycle-events) @@ -52,7 +53,7 @@ SDK MQTT5 support comes from a separate client implementation. In doing so, we * [IoT Core specific validation](https://awslabs.github.io/aws-crt-python/api/mqtt5.html#awscrt.mqtt5.ExtendedValidationAndFlowControlOptions) - will validate and fail operations that break IoT Core specific restrictions * [IoT Core specific flow control](https://awslabs.github.io/aws-crt-python/api/mqtt5.html#awscrt.mqtt5.ExtendedValidationAndFlowControlOptions) - will apply flow control to honor IoT Core specific per-connection limits and quotas * [Flexible queue control](https://awslabs.github.io/aws-crt-python/api/mqtt5.html#awscrt.mqtt5.ClientOperationQueueBehaviorType) - provides a number of options to control what happens to incomplete operations on a disconnection event -* A [new API](https://awslabs.github.io/aws-crt-python/api/mqtt5.html#awscrt.mqtt5.Client) has been added to query the internal state of the client's operation queue. This API allows the user to make more informed flow control decisions before submitting operatons to the client. +* A [new API](https://awslabs.github.io/aws-crt-python/api/mqtt5.html#awscrt.mqtt5.Client) has been added to query the internal state of the client's operation queue. This API allows the user to make more informed flow control decisions before submitting operations to the client. * Data can no longer back up on the socket. At most one frame of data is ever pending-write on the socket. * The MQTT5 client has a single message-received callback. Per-subscription callbacks are not supported. @@ -77,7 +78,8 @@ We strongly recommend using the AwsIotMqtt5ClientConfigBuilder class to configur All lifecycle events and the callback for publishes received by the MQTT5 Client should be added to the builder on creation of the Client. A full list of accepted arguments can be found in the API guide. #### **Direct MQTT with X509-based mutual TLS** For X509 based mutual TLS, you can create a client where the certificate and private key are configured by path: -``` + +```python # X.509 based certificate file certificate_file_path = "" # PKCS#1 or PKCS#8 PEM encoded private key file @@ -87,7 +89,7 @@ For X509 based mutual TLS, you can create a client where the certificate and pri # Create an MQTT5 Client using mqtt5_client_builder client = mqtt5_client_builder.mtls_from_path( - endpoint = , + endpoint = "", cert_filepath=certificate_file_path, pri_key_filepath=private_key_filePath)) ``` @@ -98,18 +100,20 @@ will sign the websocket upgrade request made by the client while connecting. Th the SDK is capable of resolving credentials in a variety of environments according to a chain of priorities: ```Environment -> Profile (local file system) -> STS Web Identity -> IMDS (ec2) or ECS``` + If the default credentials provider chain and built-in AWS region extraction logic are sufficient, you do not need to specify any additional configuration: -``` + +```python # The signing region. e.x.: 'us-east-1' - signing_region = - credentials_provider = auth.AwsCredentialsProvider.new_dfault_chain() + signing_region = "" + credentials_provider = auth.AwsCredentialsProvider.new_default_chain() # other builder configurations can be added using **kwargs in the builder # Create an MQTT5 Client using mqtt5_client_builder client = mqtt5_client_builder.websockets_with_default_aws_signing( - endpoint = , + endpoint = "", region = signing_region, credentials_provider=credentials_provider)) ``` @@ -118,37 +122,41 @@ any additional configuration: AWS IoT Core Custom Authentication allows you to use a lambda to gate access to IoT Core resources. For this authentication method, you must supply an additional configuration structure containing fields relevant to AWS IoT Core Custom Authentication. If your custom authenticator does not use signing, you don't specify anything related to the token signature: -``` + +```python # other builder configurations can be added using **kwargs in the builder client = mqtt5_client_builder.direct_with_custom_authorizer( - endpoint = , - auth_authorizer_name = , - auth_username = , - auth_password =) + endpoint = "", + auth_authorizer_name = "", + auth_username = "", + auth_password = ) ``` + If your custom authorizer uses signing, you must specify the three signed token properties as well. The token signature must be the URI-encoding of the base64 encoding of the digital signature of the token value via the private key associated with the public key that was registered with the custom authorizer. It is your responsibility to URI-encode the token signature. -``` + +```python # other builder configurations can be added using **kwargs in the builder client = mqtt5_client_builder.direct_with_custom_authorizer( - endpoint = , - auth_authorizer_name = , - auth_username = , - auth_password =, - auth_authorizer_signature=) + endpoint = "", + auth_authorizer_name = "", + auth_username = "", + auth_password = , + auth_authorizer_signature= "") ``` + In both cases, the builder will construct a final CONNECT packet username field value for you based on the values configured. Do not add the token-signing fields to the value of the username that you assign within the custom authentication config structure. Similarly, do not add any custom authentication related values to the username in the CONNECT configuration optionally attached to the client configuration. The builder will do everything for you. #### **Direct MQTT with PKCS11 Method** A MQTT5 direct connection can be made using a PKCS11 device rather than using a PEM encoded private key, the private key for mutual TLS is stored on a PKCS#11 compatible smart card or Hardware Security Module (HSM). To create a MQTT5 builder configured for this connection, see the following code: -``` +```python # other builder configurations can be added using **kwargs in the builder pkcs11_lib = io.Pkcs11Lib( - file=, + file="", behavior=io.Pkcs11Lib.InitializeFinalizeBehavior.STRICT) client = mqtt5_client_builder.mtls_with_pkcs11( @@ -158,26 +166,57 @@ A MQTT5 direct connection can be made using a PKCS11 device rather than using a token_label=pkcs11_token_label, priave_key_label=pkcs11_private_key_label, cert_filepath=pkcs11_cert_filepath, - endpoint = ) + endpoint = "") ``` **Note**: Currently, TLS integration with PKCS#11 is only available on Unix devices. +#### **MQTT over Websockets with Cognito authentication** + +A MQTT5 websocket connection can be made using Cognito to authenticate rather than the AWS credentials located on the device or via key and certificate. Instead, Cognito can authenticate the connection using a valid Cognito identity ID. This requires a valid Cognito identity ID, which can be retrieved from a Cognito identity pool. A Cognito identity pool can be created from the AWS console. + +To create a MQTT5 builder configured for this connection, see the following code: + +```python + # The signing region. e.x.: 'us-east-1' + signing_region = "" + + # See https://docs.aws.amazon.com/general/latest/gr/cognito_identity.html for Cognito endpoints + cognito_endpoint = "cognito-identity." + signing_region + ".amazonaws.com" + cognito_identity_id = "" + credentials_provider = auth.AwsCredentialsProvider.new_cognito( + endpoint=cognito_endpoint, + identity=cognito_identity_id, + tls_ctx=io.ClientTlsContext(TlsContextOptions())) + + # other builder configurations can be added using **kwargs in the builder + + # Create an MQTT5 Client using mqtt5_client_builder + client = mqtt5_client_builder.websockets_with_default_aws_signing( + endpoint = "", + region = signing_region, + credentials_provider=credentials_provider)) +``` + +**Note**: A Cognito identity ID is different from a Cognito identity pool ID and trying to connect with a Cognito identity pool ID will not work. If you are unable to connect, make sure you are passing a Cognito identity ID rather than a Cognito identity pool ID. + #### **HTTP Proxy** No matter what your connection transport or authentication method is, you may connect through an HTTP proxy by adding the http_proxy_options keyword argument to the builder: -``` + +```python http_proxy_options = http.HttpProxyOptions( - host_name = , + host_name = "", port = ) # Create an MQTT5 Client using mqtt5_client_builder with proxy options as keyword argument client = mqtt5_client_builder.mtls_from_path( - endpoint = , - cert_filepath = , - pri_key_filepath = , + endpoint = "", + cert_filepath = "", + pri_key_filepath = "", http_proxy_options = http_proxy_options)) ``` + SDK Proxy support also includes support for basic authentication and TLS-to-proxy. SDK proxy support does not include any additional proxy authentication methods (kerberos, NTLM, etc...) nor does it include non-HTTP proxies (SOCKS5, for example). @@ -185,11 +224,10 @@ proxy authentication methods (kerberos, NTLM, etc...) nor does it include non-HT Once created, an MQTT5 client's configuration is immutable. Invoking start() on the client will put it into an active state where it recurrently establishes a connection to the configured remote endpoint. Reconnecting continues until you invoke stop(). -``` -# Create an MQTT5 Client - +```python + # Create an MQTT5 Client client_options = mqtt5.ClientOptions( - host_name = , + host_name = "", port = ) # Other options in client options can be set but once Client is initialized configuration is immutable @@ -198,16 +236,15 @@ recurrently establishes a connection to the configured remote endpoint. Reconne client = mqtt5.Client(client_options) - -# Use the client + # Use the client client.start(); ... ``` Invoking stop() breaks the current connection (if any) and moves the client into an idle state. -``` - // Shutdown +```python + # Shutdown client.stop(); ``` @@ -227,14 +264,15 @@ Emitted when a connection attempt fails at any point between DNS resolution and Emitted when the client's network connection is shut down, either by a local action, event, or a remote close or reset. Only emitted after a ConnectionSuccess event: a network connection that is shut down during the connecting process manifests as a ConnectionFailure event. A Disconnect event will always include an error code. If the Disconnect event is due to the receipt of a server-sent DISCONNECT packet, the packet will be included with the event data. #### **Stopped** -Emitted once the client has shutdown any associated network connection and entered an idle state where it will no longer attempt to reconnect. Only emitted after an invocation of stop() on the client. A stopped client may always be started again. +Emitted once the client has shutdown any associated network connection and entered an idle state where it will no longer attempt to reconnect. Only emitted after an invocation of `stop()` on the client. A stopped client may always be started again. ## **Client Operations** There are four basic MQTT operations you can perform with the MQTT5 client. ### Subscribe The Subscribe operation takes a description of the SUBSCRIBE packet you wish to send and returns a future that resolves successfully with the corresponding SUBACK returned by the broker; the future result raises an exception if anything goes wrong before the SUBACK is received. -``` + +```python subscribe_future = client.subscribe(subscribe_packet = mqtt5.SubscribePacket( subscriptions = [mqtt5.Subscription( topic_filter = "hello/world/qos1", @@ -242,17 +280,21 @@ The Subscribe operation takes a description of the SUBSCRIBE packet you wish to suback = subscribe_future.result() ``` + ### Unsubscribe The Unsubscribe operation takes a description of the UNSUBSCRIBE packet you wish to send and returns a future that resolves successfully with the corresponding UNSUBACK returned by the broker; the future result raises an exception if anything goes wrong before the UNSUBACK is received. -``` + +```python unsubscribe_future = client.unsubscribe(unsubscribe_packet = mqtt5.UnsubscribePacket( topic_filters=["hello/world/qos1"])) unsuback = unsubscribe_future.result() ``` + ### Publish The Publish operation takes a description of the PUBLISH packet you wish to send and returns a future of polymorphic value. If the PUBLISH was a QoS 0 publish, then the future result is an empty PUBACK packet with all members set to None and is completed as soon as the packet has been written to the socket. If the PUBLISH was a QoS 1 publish, then the future result is a PUBACK packet value and is completed as soon as the PUBACK is received from the broker. If the operation fails for any reason before these respective completion events, the future result raises an exception. -``` + +```python publish_future = client.publish(mqtt5.PublishPacket( topic = "hello/world/qos1", payload = "This is the payload of a QoS 1 publish", @@ -261,9 +303,11 @@ The Publish operation takes a description of the PUBLISH packet you wish to send # on success, the result of publish_future will be a PubackPacket puback = publish_future.result() ``` + ### Disconnect -The stop() API supports a DISCONNECT packet as an optional parameter. If supplied, the DISCONNECT packet will be sent to the server prior to closing the socket. There is no future returned by a call to stop() but you may listen for the 'stopped' event on the client. -``` +The `stop()` API supports a DISCONNECT packet as an optional parameter. If supplied, the DISCONNECT packet will be sent to the server prior to closing the socket. There is no future returned by a call to `stop()` but you may listen for the 'stopped' event on the client. + +```python client.stop(mqtt5.DisconnectPacket( reason_code = mqtt5.DisconnectReasonCode.NORMAL_DISCONNECTION, session_expiry_interval_sec = 3600)) @@ -277,4 +321,4 @@ Below are some best practices for the MQTT5 client that are recommended to follo * Use the minimum QoS you can get away with for the lowest latency and bandwidth costs. For example, if you are sending data consistently multiple times per second and do not have to have a guarantee the server got each and every publish, using QoS 0 may be ideal compared to QoS 1. Of course, this heavily depends on your use case but generally it is recommended to use the lowest QoS possible. * If you are getting unexpected disconnects when trying to connect to AWS IoT Core, make sure to check your IoT Core Thing’s policy and permissions to make sure your device is has the permissions it needs to connect! * For **Publish**, **Subscribe**, and **Unsubscribe**, you can check the reason codes in the returned Future to see if the operation actually succeeded. -* You MUST NOT perform blocking operations on any callback, or you will cause a deadlock. For example: in the on_publish_received callback, do not send a publish, and then wait for the future to complete within the callback. The Client cannot do work until your callback returs, so the thread will be stuck. \ No newline at end of file +* You MUST NOT perform blocking operations on any callback, or you will cause a deadlock. For example: in the `on_publish_received` callback, do not send a publish, and then wait for the future to complete within the callback. The Client cannot do work until your callback returns, so the thread will be stuck. diff --git a/samples/README.md b/samples/README.md index 7454d0ba..6344ee3c 100644 --- a/samples/README.md +++ b/samples/README.md @@ -7,6 +7,7 @@ * [PKCS#11 Connect](#pkcs11-connect) * [Windows Certificate Connect](#windows-certificate-connect) * [Custom Authorizer Connect](#custom-authorizer-connect) +* [Cognito Connect](#cognito-connect) * [Shadow](#shadow) * [Jobs](#jobs) * [Fleet Provisioning](#fleet-provisioning) @@ -411,11 +412,58 @@ Your Thing's [Policy](https://docs.aws.amazon.com/iot/latest/developerguide/iot- Run the sample like this: ``` sh # For Windows: replace 'python3' with 'python' -python3 custom_authorizer_connect.py --endpoint --ca_file --custom_auth_authorizer_name +python3 custom_authorizer_connect.py --endpoint --custom_auth_authorizer_name ``` You will need to setup your Custom Authorizer so that the lambda function returns a policy document. See [this page on the documentation](https://docs.aws.amazon.com/iot/latest/developerguide/config-custom-auth.html) for more details and example return result. +## Cognito Connect + +This sample makes an MQTT websocket connection and connects through a [Cognito](https://aws.amazon.com/cognito/) identity. On startup, the device connects to the server and then disconnects. This sample is for reference on connecting using Cognito. + +To run this sample, you need to have a Cognito identifier ID. You can get a Cognito identifier ID by creating a Cognito identity pool. For creating Cognito identity pools, please see the following page on the AWS documentation: [Tutorial: Creating an identity pool](https://docs.aws.amazon.com/cognito/latest/developerguide/tutorial-create-identity-pool.html) + +**Note:** This sample assumes using an identity pool with unauthenticated identity access for the sake of convenience. Please follow best practices in a real world application based on the needs of your application and the intended use case. + +Once you have a Cognito identity pool, you can run the following CLI command to get the Cognito identity pool ID: +```sh +aws cognito-identity get-id --identity-pool-id +# result from above command +{ + "IdentityId": "" +} +``` + +You can then use the returned ID in the `IdentityId` result as the input for the `--cognito_identity` argument. Please note that the Cognito identity pool ID is **not** the same as a Cognito identity ID and the sample will not work if you pass a Cognito pool id. + +Your IoT Thing's [Policy](https://docs.aws.amazon.com/iot/latest/developerguide/iot-policies.html) must provide privileges for this sample to connect. Make sure your policy allows a client ID of `test-*` to connect or use `--client_id ` to send the client ID your policy supports. + +
+(see sample policy) +
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Action": [
+        "iot:Connect"
+      ],
+      "Resource": [
+        "arn:aws:iot:region:account:client/test-*"
+      ]
+    }
+  ]
+}
+
+
+ +Run the sample like this: +``` sh +# For Windows: replace 'python3' with 'python' +python3 cognito_connect.py --endpoint --signing_region --cognito_identity +``` + ## Shadow This sample uses the AWS IoT diff --git a/samples/cognito_connect.py b/samples/cognito_connect.py new file mode 100644 index 00000000..4e00221f --- /dev/null +++ b/samples/cognito_connect.py @@ -0,0 +1,66 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0. + +from uuid import uuid4 + +# This sample shows how to create a MQTT connection using Cognito. +# This sample is intended to be used as a reference for making MQTT connections. + +# Parse arguments +import command_line_utils +cmdUtils = command_line_utils.CommandLineUtils("Cognito Connect - Make a Cognito MQTT connection.") +cmdUtils.add_common_mqtt_commands() +cmdUtils.add_common_proxy_commands() +cmdUtils.add_common_logging_commands() +cmdUtils.register_command("signing_region", "", + "The signing region used for the websocket signer", + True, str) +cmdUtils.register_command("client_id", "", + "Client ID to use for MQTT connection (optional, default='test-*').", + default="test-" + str(uuid4())) +cmdUtils.register_command("cognito_identity", "", + "The Cognito identity ID to use to connect via Cognito", + True, str) +cmdUtils.register_command("is_ci", "", "If present the sample will run in CI mode (optional, default='None')") +# Needs to be called so the command utils parse the commands +cmdUtils.get_args() +is_ci = cmdUtils.get_command("is_ci", None) is not None + +# Callback when connection is accidentally lost. +def on_connection_interrupted(connection, error, **kwargs): + print("Connection interrupted. error: {}".format(error)) + +# Callback when an interrupted connection is re-established. +def on_connection_resumed(connection, return_code, session_present, **kwargs): + print("Connection resumed. return_code: {} session_present: {}".format(return_code, session_present)) + + +if __name__ == '__main__': + # Create a connection using Cognito. + # Note: The data for the connection is gotten from cmdUtils. + # (see build_cognito_mqtt_connection for implementation) + # + # Note: This sample and code assumes that you are using a Cognito identity + # in the same region as you pass to "--signing_region". + # If not, you may need to adjust the Cognito endpoint in the cmdUtils. + # See https://docs.aws.amazon.com/general/latest/gr/cognito_identity.html + # for all Cognito endpoints. + mqtt_connection = cmdUtils.build_cognito_mqtt_connection(on_connection_interrupted, on_connection_resumed) + + if not is_ci: + print("Connecting to {} with client ID '{}'...".format( + cmdUtils.get_command(cmdUtils.m_cmd_endpoint), cmdUtils.get_command("client_id"))) + else: + print("Connecting to endpoint with client ID...") + + connect_future = mqtt_connection.connect() + + # Future.result() waits until a result is available + connect_future.result() + print("Connected!") + + # Disconnect + print("Disconnecting...") + disconnect_future = mqtt_connection.disconnect() + disconnect_future.result() + print("Disconnected!") diff --git a/samples/command_line_utils.py b/samples/command_line_utils.py index 3da00647..dc8af5c2 100644 --- a/samples/command_line_utils.py +++ b/samples/command_line_utils.py @@ -212,6 +212,28 @@ def build_websocket_mqtt_connection(self, on_connection_interrupted, on_connecti keep_alive_secs=30) return mqtt_connection + def build_cognito_mqtt_connection(self, on_connection_interrupted, on_connection_resumed): + proxy_options = self.get_proxy_options_for_mqtt_connection() + + cognito_endpoint = "cognito-identity." + self.get_command_required(self.m_cmd_signing_region) + ".amazonaws.com" + credentials_provider = auth.AwsCredentialsProvider.new_cognito( + endpoint=cognito_endpoint, + identity=self.get_command_required(self.m_cmd_cognito_identity), + tls_ctx=io.ClientTlsContext(io.TlsContextOptions())) + + mqtt_connection = mqtt_connection_builder.websockets_with_default_aws_signing( + endpoint=self.get_command_required(self.m_cmd_endpoint), + region=self.get_command_required(self.m_cmd_signing_region), + credentials_provider=credentials_provider, + http_proxy_options=proxy_options, + ca_filepath=self.get_command(self.m_cmd_ca_file), + on_connection_interrupted=on_connection_interrupted, + on_connection_resumed=on_connection_resumed, + client_id=self.get_command_required("client_id"), + clean_session=False, + keep_alive_secs=30) + return mqtt_connection + def build_direct_mqtt_connection(self, on_connection_interrupted, on_connection_resumed): proxy_options = self.get_proxy_options_for_mqtt_connection() mqtt_connection = mqtt_connection_builder.mtls_from_path( @@ -378,3 +400,4 @@ def build_mqtt5_client(self, m_cmd_custom_auth_authorizer_name = "custom_auth_authorizer_name" m_cmd_custom_auth_authorizer_signature = "custom_auth_authorizer_signature" m_cmd_custom_auth_password = "custom_auth_password" + m_cmd_cognito_identity = "cognito_identity"