From 8ee74da386b58f2b926398f5684533e205eece0d Mon Sep 17 00:00:00 2001 From: Steve Kim <86316075+sbSteveK@users.noreply.github.com> Date: Tue, 1 Nov 2022 11:19:28 -0700 Subject: [PATCH 01/61] Mqtt5 sync (#11) MQTT5 Builder and Sample --- awsiot/__init__.py | 1 + awsiot/mqtt5_client_builder.py | 636 +++++++++++++++++++++++++ docsrc/awsiot/mqtt5_client_builder.rst | 4 + docsrc/conf.py | 2 +- docsrc/index.rst | 1 + samples/command_line_utils.py | 203 +++++++- samples/mqtt5_pubsub.py | 150 ++++++ 7 files changed, 983 insertions(+), 14 deletions(-) create mode 100644 awsiot/mqtt5_client_builder.py create mode 100644 docsrc/awsiot/mqtt5_client_builder.rst create mode 100644 samples/mqtt5_pubsub.py diff --git a/awsiot/__init__.py b/awsiot/__init__.py index accc4c46..6a65102c 100644 --- a/awsiot/__init__.py +++ b/awsiot/__init__.py @@ -8,6 +8,7 @@ 'iotshadow', 'greengrass_discovery', 'mqtt_connection_builder', + 'mqtt5_client_builder', ] from awscrt import mqtt diff --git a/awsiot/mqtt5_client_builder.py b/awsiot/mqtt5_client_builder.py new file mode 100644 index 00000000..973bf562 --- /dev/null +++ b/awsiot/mqtt5_client_builder.py @@ -0,0 +1,636 @@ +""" +Builder functions to create a :class:`awscrt.mqtt5.Client`, configured for use with AWS IoT Core. +The following keyword arguments are common to all builder functions: + +Required Keyword Arguments: + + **endpoint** (`str`): Host name of AWS IoT server. + +Optional Keyword Arguments (omit, or set `None` to get default value): + **client_options** (:class:`awscrt.mqtt5.ClientOptions`): This dataclass can be used to to apply all + configuration options for Client creation. Any options set within will supercede defaults + assigned by the builder. Any omitted arguments within this class will be filled by additional + keyword arguments provided to the builder or be set to their default values. + + **connect_options** (:class:`awscrt.mqtt5.ConnectPacket`): This dataclass can be used to apply connection + options for the client. Any options set within will supercede defaults assigned by the builder but + will not overwrite options set by connect_options included within a client_options keyword argument. + Any omitted arguments within this class will be assigned values of keyword arguments provided to + the builder. + + **client_id** (`str`): ID to place in CONNECT packet. Must be unique across all devices/clients. + If an ID is already in use, the other client will be disconnected. If one is not provided, + AWS IoT server will assign a unique ID for use and return it in the CONNACK packet. + + **port** (`int`): Override default server port. + Default port is 443 if system supports ALPN or websockets are being used. + Otherwise, default port is 8883. + + **client_bootstrap** (:class:`awscrt.io.ClientBootstrap`): Client bootstrap used to establish connection. + The ClientBootstrap will default to the static default (Io.ClientBootstrap.get_or_create_static_default) + if the argument is omitted or set to 'None'. + + **http_proxy_options** (:class:`awscrt.http.HttpProxyOptions`): HTTP proxy options to use + + **keep_alive_interval_sec** (`int`): The maximum time interval, in seconds, that is permitted to elapse + between the point at which the client finishes transmitting one MQTT packet and the point it starts + sending the next. The client will use PINGREQ packets to maintain this property. If the responding + CONNACK contains a keep alive property value, then that is the negotiated keep alive value. Otherwise, + the keep alive sent by the client is the negotiated value. + + **username** (`str`): Username to connect with. + + **password** (`str`): Password to connect with. + + **session_expiry_interval_sec** (`int`): A time interval, in seconds, that the client requests the server + to persist this connection's MQTT session state for. Has no meaning if the client has not been + configured to rejoin sessions. Must be non-zero in order to successfully rejoin a session. If the + responding CONNACK contains a session expiry property value, then that is the negotiated session + expiry value. Otherwise, the session expiry sent by the client is the negotiated value. + + **request_response_information** (`bool`): If true, requests that the server send response information in + the subsequent CONNACK. This response information may be used to set up request-response implementations + over MQTT, but doing so is outside the scope of the MQTT5 spec and client. + + **request_problem_information** (`bool`): If true, requests that the server send additional diagnostic + information (via response string or user properties) in DISCONNECT or CONNACK packets from the server. + + **receive_maximum** (`int`): Notifies the server of the maximum number of in-flight QoS 1 and 2 messages the + client is willing to handle. If omitted or null, then no limit is requested. + + **maximum_packet_size** (`int`): Notifies the server of the maximum packet size the client is willing to handle. + If omitted or null, then no limit beyond the natural limits of MQTT packet size is requested. + + **will_delay_interval_sec** (`int`): A time interval, in seconds, that the server should wait (for a session + reconnection) before sending the will message associated with the connection's session. If omitted or + null, the server will send the will when the associated session is destroyed. If the session is destroyed + before a will delay interval has elapsed, then the will must be sent at the time of session destruction. + + **will** (:class:`awscrt.mqtt5.PublishPacket`): The definition of a message to be published when the connection's + session is destroyed by the server or when the will delay interval has elapsed, whichever comes first. If + null, then nothing will be sent. + + **user_properties** (`Sequence` [:class:`awscrt.mqtt5.UserProperty`]): List of MQTT5 user properties included + with the packet. + + **session_behavior** (:class:`awscrt.mqtt5.ClientSessionBehaviorType`): How the MQTT5 client should behave with + respect to MQTT sessions. + + **extended_validation_and_flow_control_options** (:class:`awscrt.mqtt5.ExtendedValidationAndFlowControlOptions`): + The additional controls for client behavior with respect to operation validation and flow control; these + checks go beyond the base MQTT5 spec to respect limits of specific MQTT brokers. If argument is omitted or null, + then set to AWS_IOT_CORE_DEFAULTS. + + **offline_queue_behavior** (:class:`awscrt.mqtt5.ClientOperationQueueBehaviorType`): Returns how disconnects + affect the queued and in-progress operations tracked by the client. Also controls how new operations are + handled while the client is not connected. In particular, if the client is not connected, then any operation + that would be failed on disconnect (according to these rules) will also be rejected. + + **retry_jitter_mode** (:class:`awscrt.mqtt5.ExponentialBackoffJitterMode`): How the reconnect delay is modified + in order to smooth out the distribution of reconnection attempt timepoints for a large set of reconnecting + clients. + + **min_reconnect_delay_ms** (`int`): The minimum amount of time to wait to reconnect after a disconnect. + Exponential backoff is performed with jitter after each connection failure. + + **max_reconnect_delay_ms** (`int`): The maximum amount of time to wait to reconnect after a disconnect. + Exponential backoff is performed with jitter after each connection failure. + + **min_connected_time_to_reset_reconnect_delay_ms** (`int`): The amount of time that must elapse with an + established connection before the reconnect delay is reset to the minimum. This helps alleviate + bandwidth-waste in fast reconnect cycles due to permission failures on operations. + + **ping_timeout_ms** (`int`): The time interval to wait after sending a PINGREQ for a PINGRESP to arrive. If one + does not arrive, the client will close the current connection. + + **connack_timeout_ms** (`int`): The time interval to wait after sending a CONNECT request for a CONNACK to arrive. + If one does not arrive, the connection will be shut down. + + **operation_timeout_sec** (`int`): The time interval to wait for an ack after sending a QoS 1+ PUBLISH, SUBSCRIBE, + or UNSUBSCRIBE before failing the operation. + + **on_publish_received** (`Callable`): Callback invoked for all publish packets received by client. + The function should take the following arguments and return nothing: + + * `publish_packet` (:class:`awscrt.mqtt5.PublishPacket`): Publish Packet received from the server. + + **on_lifecycle_stopped** (`Callable`): Callback invoked for Lifecycle Event Stopped. + The function should take the following arguments and return nothing: + + * `lifecycle_stopped_data` (:class:`awscrt.mqtt5.LifecycleStoppedData`): Currently unused dataclass. + + **on_lifecycle_attempting_connect** (`Callable`): Callback invoked for Lifecycle Event Attempting Connect. + The function should take the following arguments and return nothing: + + * `lifecycle_attempting_connect_data` (:class:`awscrt.mqtt5.LifecycleAttemptingConnectData`): Currently + unused dataclass. + + **on_lifecycle_connection_success** (`Callable`): Callback invoked for Lifecycle Event Connection Success. + The function should take the following arguments and return nothing: + + * `lifecycle_connect_success_data` (:class:`awscrt.mqtt5.LifecycleConnectSuccessData`): Dataclass + containing the following: + + * `connack_packet` (:class:`awscrt.mqtt5.ConnackPacket`): Data model of an `MQTT5 CONNACK `_ packet. + + * `negotiated_settings` (:class:`awscrt.mqtt5.NegotiatedSettings`): Mqtt behavior settings that have been dynamically negotiated as part of the CONNECT/CONNACK exchange. + + **on_lifecycle_connection_failure** (`Callable`): Callback invoked for Lifecycle Event Connection Failure. + The function should take the following arguments and return nothing: + + * `lifecycle_connection_failure_data` (:class:`awscrt.mqtt5.LifecycleConnectFailureData`): Dataclass + containing the following: + + * `connack_packet` (:class:`awscrt.mqtt5.ConnackPacket`): Data model of an `MQTT5 CONNACK `_ packet. + + * `error_code` (`int`): Exception which caused connection failure. + + **on_lifecycle_disconnection** (`Callable`): Callback invoked for Lifecycle Event Disconnection. + The function should take the following arguments and return nothing: + + * `lifecycle_disconnect_data` (:class:`awscrt.mqtt5.LifecycleDisconnectData`): Dataclass + containing the following: + + * `disconnect_packet` (:class:`awscrt.mqtt5.DisconnectPacket`): Data model of an `MQTT5 DISCONNECT `_ packet. + + * `error_code` (`int`): Exception which caused disconnection. + + **ca_filepath** (`str`): Override default trust store with CA certificates from this PEM formatted file. + + **ca_dirpath** (`str`): Override default trust store with CA certificates loaded from this directory (Unix only). + + **ca_bytes** (`bytes`): Override default trust store with CA certificates from these PEM formatted bytes. + + **enable_metrics_collection** (`bool`): Whether to send the SDK version number in the CONNECT packet. + Default is True. + + +""" + +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0. + +import awscrt.auth +import awscrt.io +import awscrt.mqtt5 + +DEFAULT_WEBSOCKET_MQTT_PORT = 443 +DEFAULT_DIRECT_MQTT_PORT = 8883 +DEFAULT_KEEP_ALIVE = 1200 + + +def _check_required_kwargs(**kwargs): + for required in ['endpoint']: + if not kwargs.get(required): + raise TypeError("Builder needs keyword-only argument '{}'".format(required)) + + +def _get(kwargs, name, default=None): + """ + Returns kwargs['name'] if it exists and is not None. + Otherwise returns default. + + This function exists so users can pass some_arg=None to get its default + value, instead of literally passing None. + """ + val = kwargs.get(name) + if val is None: + val = default + return val + + +_metrics_str = None + + +def _get_metrics_str(current_username=""): + global _metrics_str + + username_has_query = False + if current_username.find("?") != -1: + username_has_query = True + + if _metrics_str is None: + try: + import pkg_resources + try: + version = pkg_resources.get_distribution("awsiotsdk").version + _metrics_str = "SDK=PythonV2&Version={}".format(version) + except pkg_resources.DistributionNotFound: + _metrics_str = "SDK=PythonV2&Version=dev" + except BaseException: + _metrics_str = "" + + if not _metrics_str == "": + if username_has_query: + return "&" + _metrics_str + else: + return "?" + _metrics_str + else: + return "" + + +def _builder( + tls_ctx_options, + use_websockets=False, + websocket_handshake_transform=None, + use_custom_authorizer=False, + **kwargs): + + username = _get(kwargs, 'username', '') + if _get(kwargs, 'enable_metrics_collection', True): + username += _get_metrics_str(username) + + client_options = _get(kwargs, 'client_options') + if client_options is None: + client_options = awscrt.mqtt5.ClientOptions( + host_name=_get(kwargs, 'endpoint') + ) + if client_options.connect_options is None: + client_options.connect_options = _get(kwargs, 'connect_options', awscrt.mqtt5.ConnectPacket()) + + # Client Options + if client_options.port is None: + client_options.port = _get(kwargs, 'port') + if client_options.bootstrap is None: + client_options.bootstrap = _get(kwargs, 'client_bootstrap') + if client_options.socket_options is None: + client_options.socket_options = _get(kwargs, 'socket_options') + if client_options.http_proxy_options is None: + client_options.http_proxy_options = kwargs.get( + 'http_proxy_options', kwargs.get( + 'websocket_proxy_options', None)) + if client_options.session_behavior is None: + client_options.session_behavior = _get(kwargs, 'session_behavior') + if client_options.extended_validation_and_flow_control_options is None: + client_options.extended_validation_and_flow_control_options = _get( + kwargs, + 'extended_validation_and_flow_control_options', + default=awscrt.mqtt5.ExtendedValidationAndFlowControlOptions.AWS_IOT_CORE_DEFAULTS) + if client_options.offline_queue_behavior is None: + client_options.offline_queue_behavior = _get(kwargs, 'offline_queue_behavior') + if client_options.retry_jitter_mode is None: + client_options.retry_jitter_mode = _get(kwargs, 'retry_jitter_mode') + if client_options.min_reconnect_delay_ms is None: + client_options.min_reconnect_delay_ms = _get(kwargs, 'min_reconnect_delay_ms') + if client_options.max_reconnect_delay_ms is None: + client_options.max_reconnect_delay_ms = _get(kwargs, 'max_reconnect_delay_ms') + if client_options.min_connected_time_to_reset_reconnect_delay_ms is None: + client_options.min_connected_time_to_reset_reconnect_delay_ms = _get( + kwargs, 'min_connected_time_to_reset_reconnect_delay_ms') + if client_options.ping_timeout_ms is None: + client_options.ping_timeout_ms = _get(kwargs, 'ping_timeout_ms') + if client_options.connack_timeout_ms is None: + client_options.connack_timeout_ms = _get(kwargs, 'connack_timeout_ms') + if client_options.operation_timeout_sec is None: + client_options.operation_timeout_sec = _get(kwargs, 'operation_timeout_sec') + if client_options.websocket_handshake_transform is None: + client_options.websocket_handshake_transform = websocket_handshake_transform + + # Connect Options + if client_options.connect_options.client_id is None: + client_options.connect_options.client_id = _get(kwargs, 'client_id') + if client_options.connect_options.keep_alive_interval_sec is None: + client_options.connect_options.keep_alive_interval_sec = _get( + kwargs, 'keep_alive_interval_sec', DEFAULT_KEEP_ALIVE) + client_options.connect_options.username = username + if client_options.connect_options.password is None: + client_options.connect_options.password = _get(kwargs, 'password') + if client_options.connect_options.session_expiry_interval_sec is None: + client_options.connect_options.session_expiry_interval_sec = _get(kwargs, 'session_expiry_interval_sec') + if client_options.connect_options.request_response_information is None: + client_options.connect_options.request_response_information = _get(kwargs, 'request_response_information') + if client_options.connect_options.request_problem_information is None: + client_options.connect_options.request_problem_information = _get(kwargs, 'request_problem_information') + if client_options.connect_options.receive_maximum is None: + client_options.connect_options.receive_maximum = _get(kwargs, 'receive_maximum') + if client_options.connect_options.maximum_packet_size is None: + client_options.connect_options.maximum_packet_size = _get(kwargs, 'maximum_packet_size') + if client_options.connect_options.will_delay_interval_sec is None: + client_options.connect_options.will_delay_interval_sec = _get(kwargs, 'will_delay_interval_sec') + if client_options.connect_options.will is None: + client_options.connect_options.will = _get(kwargs, 'will') + if client_options.connect_options.user_properties is None: + client_options.connect_options.user_properties = _get(kwargs, 'user_properties') + + # Callbacks + if client_options.on_publish_callback_fn is None: + client_options.on_publish_callback_fn = _get(kwargs, 'on_publish_received') + if client_options.on_lifecycle_event_stopped_fn is None: + client_options.on_lifecycle_event_stopped_fn = _get(kwargs, 'on_lifecycle_stopped') + if client_options.on_lifecycle_event_attempting_connect_fn is None: + client_options.on_lifecycle_event_attempting_connect_fn = _get(kwargs, 'on_lifecycle_attempting_connect') + if client_options.on_lifecycle_event_connection_success_fn is None: + client_options.on_lifecycle_event_connection_success_fn = _get(kwargs, 'on_lifecycle_connection_success') + if client_options.on_lifecycle_event_connection_failure_fn is None: + client_options.on_lifecycle_event_connection_failure_fn = _get(kwargs, 'on_lifecycle_connection_failure') + if client_options.on_lifecycle_event_disconnection_fn is None: + client_options.on_lifecycle_event_disconnection_fn = _get(kwargs, 'on_lifecycle_disconnection') + + ca_bytes = _get(kwargs, 'ca_bytes') + ca_filepath = _get(kwargs, 'ca_filepath') + ca_dirpath = _get(kwargs, 'ca_dirpath') + if ca_bytes: + tls_ctx_options.override_default_trust_store(ca_bytes) + elif ca_filepath or ca_dirpath: + tls_ctx_options.override_default_trust_store_from_path(ca_dirpath, ca_filepath) + + if client_options.port is None: + # prefer 443, even for direct MQTT connections, since it's less likely to be blocked by firewalls + if use_websockets or awscrt.io.is_alpn_available(): + client_options.port = DEFAULT_WEBSOCKET_MQTT_PORT + else: + client_options.port = DEFAULT_DIRECT_MQTT_PORT + + if client_options.port == 443 and awscrt.io.is_alpn_available() and use_custom_authorizer is False: + tls_ctx_options.alpn_list = ['http/1.1'] if use_websockets else ['x-amzn-mqtt-ca'] + + tls_ctx = awscrt.io.ClientTlsContext(tls_ctx_options) + client_options.tls_ctx = tls_ctx + client = awscrt.mqtt5.Client(client_options=client_options) + + return client + + +def mtls_from_path(cert_filepath, pri_key_filepath, **kwargs) -> awscrt.mqtt5.Client: + """ + This builder creates an :class:`awscrt.mqtt5.Client`, configured for an mTLS MQTT5 Client to AWS IoT. + TLS arguments are passed as filepaths. + + This function takes all :mod:`common arguments` + described at the top of this doc, as well as... + + Keyword Args: + cert_filepath (str): Path to certificate file. + + pri_key_filepath (str): Path to private key file. + """ + _check_required_kwargs(**kwargs) + tls_ctx_options = awscrt.io.TlsContextOptions.create_client_with_mtls_from_path(cert_filepath, pri_key_filepath) + return _builder(tls_ctx_options, **kwargs) + + +def mtls_from_bytes(cert_bytes, pri_key_bytes, **kwargs) -> awscrt.mqtt5.Client: + """ + This builder creates an :class:`awscrt.mqtt5.Client`, configured for an mTLS MQTT5 Client to AWS IoT. + TLS arguments are passed as in-memory bytes. + + This function takes all :mod:`common arguments` + described at the top of this doc, as well as... + + Keyword Args: + cert_bytes (bytes): Certificate file bytes. + + pri_key_bytes (bytes): Private key bytes. + """ + _check_required_kwargs(**kwargs) + tls_ctx_options = awscrt.io.TlsContextOptions.create_client_with_mtls(cert_bytes, pri_key_bytes) + return _builder(tls_ctx_options, **kwargs) + + +def mtls_with_pkcs11(*, + pkcs11_lib: awscrt.io.Pkcs11Lib, + user_pin: str, + slot_id: int = None, + token_label: str = None, + private_key_label: str = None, + cert_filepath: str = None, + cert_bytes=None, + **kwargs) -> awscrt.mqtt5.Client: + """ + This builder creates an :class:`awscrt.mqtt5.Client`, configured for an mTLS MQTT connection to AWS IoT, + using a PKCS#11 library for private key operations. + + NOTE: Unix only + + This function takes all :mod:`common arguments` + described at the top of this doc, as well as... + + Args: + pkcs11_lib: Use this PKCS#11 library + + user_pin: User PIN, for logging into the PKCS#11 token. + Pass `None` to log into a token with a "protected authentication path". + + slot_id: ID of slot containing PKCS#11 token. + If not specified, the token will be chosen based on other criteria (such as token label). + + token_label: Label of the PKCS#11 token to use. + If not specified, the token will be chosen based on other criteria (such as slot ID). + + private_key_label: Label of private key object on PKCS#11 token. + If not specified, the key will be chosen based on other criteria + (such as being the only available private key on the token). + + cert_filepath: Use this X.509 certificate (file on disk). + The certificate must be PEM-formatted. The certificate may be + specified by other means instead (ex: `cert_bytes`) + + cert_bytes (Optional[Union[str, bytes, bytearray]]): + Use this X.509 certificate (contents in memory). + The certificate must be PEM-formatted. The certificate may be + specified by other means instead (ex: `cert_filepath`) + """ + _check_required_kwargs(**kwargs) + + tls_ctx_options = awscrt.io.TlsContextOptions.create_client_with_mtls_pkcs11( + pkcs11_lib=pkcs11_lib, + user_pin=user_pin, + slot_id=slot_id, + token_label=token_label, + private_key_label=private_key_label, + cert_file_path=cert_filepath, + cert_file_contents=cert_bytes) + return _builder(tls_ctx_options, **kwargs) + + +def mtls_with_windows_cert_store_path(*, + cert_store_path: str, + **kwargs) -> awscrt.mqtt5.Client: + """ + This builder creates an :class:`awscrt.mqtt5.Client`, configured for an mTLS MQTT5 Client to AWS IoT, + using a client certificate in a Windows certificate store. + + NOTE: Windows only + + This function takes all :mod:`common arguments` + described at the top of this doc, as well as... + + Args: + cert_store_path: Path to certificate in a Windows certificate store. + The path must use backslashes and end with the certificate's thumbprint. + Example: ``CurrentUser\\MY\\A11F8A9B5DF5B98BA3508FBCA575D09570E0D2C6`` + """ + _check_required_kwargs(**kwargs) + + tls_ctx_options = awscrt.io.TlsContextOptions.create_client_with_mtls_windows_cert_store_path(cert_store_path) + return _builder(tls_ctx_options, **kwargs) + + +def websockets_with_default_aws_signing( + region, + credentials_provider, + websocket_proxy_options=None, + **kwargs) -> awscrt.mqtt5.Client: + """ + This builder creates an :class:`awscrt.mqtt5.Client`, configured for an MQTT5 Client over websockets to AWS IoT. + The websocket handshake is signed using credentials from the credentials_provider. + + This function takes all :mod:`common arguments` + described at the top of this doc, as well as... + + Keyword Args: + region (str): AWS region to use when signing. + + credentials_provider (awscrt.auth.AwsCredentialsProvider): Source of AWS credentials to use when signing. + + websocket_proxy_options (awscrt.http.HttpProxyOptions): Deprecated, + for proxy settings use `http_proxy_options` (described in + :mod:`common arguments`) + + """ + _check_required_kwargs(**kwargs) + + def _sign_websocket_handshake_request(transform_args, **kwargs): + # transform_args need to know when transform is done + try: + signing_config = awscrt.auth.AwsSigningConfig( + algorithm=awscrt.auth.AwsSigningAlgorithm.V4, + signature_type=awscrt.auth.AwsSignatureType.HTTP_REQUEST_QUERY_PARAMS, + credentials_provider=credentials_provider, + region=region, + service='iotdevicegateway', + omit_session_token=True, # IoT is weird and does not sign X-Amz-Security-Token + ) + + signing_future = awscrt.auth.aws_sign_request(transform_args.http_request, signing_config) + signing_future.add_done_callback(lambda x: transform_args.set_done(x.exception())) + except Exception as e: + transform_args.set_done(e) + + return websockets_with_custom_handshake(_sign_websocket_handshake_request, websocket_proxy_options, **kwargs) + + +def websockets_with_custom_handshake( + websocket_handshake_transform, + websocket_proxy_options=None, + **kwargs) -> awscrt.mqtt5.Client: + """ + This builder creates an :class:`awscrt.mqtt5.Client`, configured for an MQTT5 Client over websockets, + with a custom function to transform the websocket handshake request before it is sent to the server. + + This function takes all :mod:`common arguments` + described at the top of this doc, as well as... + + Keyword Args: + websocket_handshake_transform (Callable): Function to transform websocket handshake request. + If provided, function is called each time a websocket connection is attempted. + The function may modify the HTTP request before it is sent to the server. + See :class:`awscrt.mqtt.WebsocketHandshakeTransformArgs` for more info. + Function should take the following arguments and return nothing: + + * `transform_args` (:class:`awscrt.mqtt5.WebsocketHandshakeTransformArgs`): + Contains HTTP request to be transformed. Function must call + `transform_args.done()` when complete. + + * `**kwargs` (dict): Forward-compatibility kwargs. + + websocket_proxy_options (awscrt.http.HttpProxyOptions): Deprecated, + for proxy settings use `http_proxy_options` (described in + :mod:`common arguments`) + """ + _check_required_kwargs(**kwargs) + tls_ctx_options = awscrt.io.TlsContextOptions() + return _builder(tls_ctx_options=tls_ctx_options, + use_websockets=True, + websocket_handshake_transform=websocket_handshake_transform, + websocket_proxy_options=websocket_proxy_options, + **kwargs) + + +def _add_to_username_parameter(input_string, parameter_value, parameter_pretext): + """ + Helper function to add parameters to the username in the direct_with_custom_authorizer function + """ + return_string = input_string + + if return_string.find("?") != -1: + return_string += "&" + else: + return_string += "?" + + if parameter_value.find(parameter_pretext) != -1: + return return_string + parameter_value + else: + return return_string + parameter_pretext + parameter_value + + +def direct_with_custom_authorizer( + auth_username=None, + auth_authorizer_name=None, + auth_authorizer_signature=None, + auth_password=None, + **kwargs) -> awscrt.mqtt5.Client: + """ + This builder creates an :class:`awscrt.mqtt5.Client`, configured for an MQTT5 Client using a custom + authorizer. This function will set the username, port, and TLS options. + + This function takes all :mod:`common arguments` + described at the top of this doc, as well as... + + Keyword Args: + auth_username (`str`): The username to use with the custom authorizer. + If provided, the username given will be passed when connecting to the custom authorizer. + If not provided, it will check to see if a username has already been set (via username="example") + and will use that instead. + If no username has been set then no username will be sent with the MQTT connection. + + auth_authorizer_name (`str`): The name of the custom authorizer. + If not provided, then "x-amz-customauthorizer-name" will not be added with the MQTT connection. + + auth_authorizer_signature (`str`): The signature of the custom authorizer. + If not provided, then "x-amz-customauthorizer-name" will not be added with the MQTT connection. + + auth_password (`str`): The password to use with the custom authorizer. + If not provided, then no passord will be set. + """ + + _check_required_kwargs(**kwargs) + username_string = "" + + if auth_username is None: + if not _get(kwargs, "username") is None: + username_string += _get(kwargs, "username") + else: + username_string += auth_username + + if auth_authorizer_name is not None: + username_string = _add_to_username_parameter( + username_string, auth_authorizer_name, "x-amz-customauthorizer-name=") + if auth_authorizer_signature is not None: + username_string = _add_to_username_parameter( + username_string, auth_authorizer_signature, "x-amz-customauthorizer-signature=") + + kwargs["username"] = username_string + kwargs["password"] = auth_password + + tls_ctx_options = awscrt.io.TlsContextOptions() + tls_ctx_options.alpn_list = ["mqtt"] + + return _builder(tls_ctx_options=tls_ctx_options, + use_websockets=False, + use_custom_authorizer=True, + **kwargs) + + +def new_default_builder(**kwargs) -> awscrt.mqtt5.Client: + """ + This builder creates an :class:`awscrt.mqtt5.Client`, without any configuration besides the default TLS context options. + + This requires setting the client details manually by passing all the necessary data + in :mod:`common arguments` to make a connection + """ + _check_required_kwargs(kwargs) + tls_ctx_options = awscrt.io.TlsContextOptions() + return _builder(tls_ctx_options=tls_ctx_options, + use_websockets=False, + kwargs=kwargs) diff --git a/docsrc/awsiot/mqtt5_client_builder.rst b/docsrc/awsiot/mqtt5_client_builder.rst new file mode 100644 index 00000000..54ba8b5d --- /dev/null +++ b/docsrc/awsiot/mqtt5_client_builder.rst @@ -0,0 +1,4 @@ +awsiot.mqtt5_client_builder +=========================== + +.. automodule:: awsiot.mqtt5_client_builder diff --git a/docsrc/conf.py b/docsrc/conf.py index f0598b25..01eb46f6 100644 --- a/docsrc/conf.py +++ b/docsrc/conf.py @@ -50,7 +50,7 @@ 'awscrt': ('https://awslabs.github.io/aws-crt-python', None), } -# A string that determines how domain objects (e.g. functions, classes, +# A string that determines how domain objects (e.g. functions, classes, # attributes, etc.) are displayed in their table of contents entry. toc_object_entries_show_parents = 'hide' diff --git a/docsrc/index.rst b/docsrc/index.rst index cc614f57..68cb82e1 100644 --- a/docsrc/index.rst +++ b/docsrc/index.rst @@ -24,6 +24,7 @@ API Reference awsiot/greengrasscoreipc awsiot/greengrass_discovery awsiot/mqtt_connection_builder + awsiot/mqtt5_client_builder awsiot/iotidentity awsiot/iotjobs awsiot/iotshadow diff --git a/samples/command_line_utils.py b/samples/command_line_utils.py index b4d51d4a..3da00647 100644 --- a/samples/command_line_utils.py +++ b/samples/command_line_utils.py @@ -3,7 +3,8 @@ import argparse from awscrt import io, http, auth -from awsiot import mqtt_connection_builder +from awsiot import mqtt_connection_builder, mqtt5_client_builder + class CommandLineUtils: def __init__(self, description) -> None: @@ -63,25 +64,85 @@ def update_command(self, command_name, new_example_input=None, new_help_output=N self.commands[command_name]["action"] = new_action def add_common_mqtt_commands(self): - self.register_command(self.m_cmd_endpoint, "", "The endpoint of the mqtt server not including a port.", True, str) - self.register_command(self.m_cmd_ca_file, "", "Path to AmazonRootCA1.pem (optional, system trust store used by default)", False, str) + self.register_command( + self.m_cmd_endpoint, + "", + "The endpoint of the mqtt server not including a port.", + True, + str) + self.register_command( + self.m_cmd_ca_file, + "", + "Path to AmazonRootCA1.pem (optional, system trust store used by default)", + False, + str) + + def add_common_mqtt5_commands(self): + self.register_command( + self.m_cmd_endpoint, + "", + "The endpoint of the mqtt server not including a port.", + True, + str) + self.register_command( + self.m_cmd_ca_file, + "", + "Path to AmazonRootCA1.pem (optional, system trust store used by default)", + False, + str) def add_common_proxy_commands(self): - self.register_command(self.m_cmd_proxy_host, "", "Host name of the proxy server to connect through (optional)", False, str) - self.register_command(self.m_cmd_proxy_port, "", "Port of the http proxy to use (optional, default='8080')", type=int, default=8080) + self.register_command( + self.m_cmd_proxy_host, + "", + "Host name of the proxy server to connect through (optional)", + False, + str) + self.register_command( + self.m_cmd_proxy_port, + "", + "Port of the http proxy to use (optional, default='8080')", + type=int, + default=8080) def add_common_topic_message_commands(self): - self.register_command(self.m_cmd_topic, "", "Topic to publish, subscribe to (optional, default='test/topic').", default="test/topic") - self.register_command(self.m_cmd_message, "", "The message to send in the payload (optional, default='Hello World!').", default="Hello World!") + self.register_command( + self.m_cmd_topic, + "", + "Topic to publish, subscribe to (optional, default='test/topic').", + default="test/topic") + self.register_command( + self.m_cmd_message, + "", + "The message to send in the payload (optional, default='Hello World!').", + default="Hello World!") def add_common_logging_commands(self): - self.register_command(self.m_cmd_verbosity, "", "Logging level.", default=io.LogLevel.NoLogs.name, choices=[x.name for x in io.LogLevel]) + self.register_command( + self.m_cmd_verbosity, + "", + "Logging level.", + default=io.LogLevel.NoLogs.name, + choices=[ + x.name for x in io.LogLevel]) def add_common_custom_authorizer_commands(self): - self.register_command(self.m_cmd_custom_auth_username, "", "The name to send when connecting through the custom authorizer (optional)") - self.register_command(self.m_cmd_custom_auth_authorizer_name, "", "The name of the custom authorizer to connect to (optional but required for everything but custom domains)") - self.register_command(self.m_cmd_custom_auth_username, "", "The signature to send when connecting through a custom authorizer (optional)") - self.register_command(self.m_cmd_custom_auth_password, "", "The password to send when connecting through a custom authorizer (optional)") + self.register_command( + self.m_cmd_custom_auth_username, + "", + "The name to send when connecting through the custom authorizer (optional)") + self.register_command( + self.m_cmd_custom_auth_authorizer_name, + "", + "The name of the custom authorizer to connect to (optional but required for everything but custom domains)") + self.register_command( + self.m_cmd_custom_auth_username, + "", + "The signature to send when connecting through a custom authorizer (optional)") + self.register_command( + self.m_cmd_custom_auth_password, + "", + "The password to send when connecting through a custom authorizer (optional)") """ Returns the command if it exists and has been passed to the console, otherwise it will print the help for the sample and exit the application. @@ -176,9 +237,125 @@ def build_mqtt_connection(self, on_connection_interrupted, on_connection_resumed def get_proxy_options_for_mqtt_connection(self): proxy_options = None if self.parsed_commands.proxy_host and self.parsed_commands.proxy_port: - proxy_options = http.HttpProxyOptions(host_name=self.parsed_commands.proxy_host, port=self.parsed_commands.proxy_port) + proxy_options = http.HttpProxyOptions( + host_name=self.parsed_commands.proxy_host, + port=self.parsed_commands.proxy_port) return proxy_options + ######################################################################## + # MQTT5 + ######################################################################## + + def build_pkcs11_mqtt5_client(self, + on_publish_received=None, + on_lifecycle_stopped=None, + on_lifecycle_attempting_connect=None, + on_lifecycle_connection_success=None, + on_lifecycle_connection_failure=None, + on_lifecycle_disconnection=None): + + pkcs11_lib_path = self.get_command_required(self.m_cmd_pkcs11_lib) + print(f"Loading PKCS#11 library '{pkcs11_lib_path}' ...") + pkcs11_lib = io.Pkcs11Lib( + file=pkcs11_lib_path, + behavior=io.Pkcs11Lib.InitializeFinalizeBehavior.STRICT) + print("Loaded!") + + pkcs11_slot_id = None + if (self.get_command(self.m_cmd_pkcs11_slot) is not None): + pkcs11_slot_id = int(self.get_command(self.m_cmd_pkcs11_slot)) + + # Create MQTT5 client + mqtt5_client = mqtt5_client_builder.mtls_with_pkcs11( + pkcs11_lib=pkcs11_lib, + user_pin=self.get_command_required(self.m_cmd_pkcs11_pin), + slot_id=pkcs11_slot_id, + token_label=self.get_command_required(self.m_cmd_pkcs11_token), + private_key_label=self.get_command_required(self.m_cmd_pkcs11_key), + cert_filepath=self.get_command_required(self.m_cmd_pkcs11_cert), + endpoint=self.get_command_required(self.m_cmd_endpoint), + port=self.get_command("port"), + ca_filepath=self.get_command(self.m_cmd_ca_file), + on_publish_received=on_publish_received, + on_lifecycle_stopped=on_lifecycle_stopped, + on_lifecycle_attempting_connect=on_lifecycle_attempting_connect, + on_lifecycle_connection_success=on_lifecycle_connection_success, + on_lifecycle_connection_failure=on_lifecycle_connection_failure, + on_lifecycle_disconnection=on_lifecycle_disconnection, + client_id=self.get_command("client_id")) + + return mqtt5_client + + def build_websocket_mqtt5_client(self, + on_publish_received=None, + on_lifecycle_stopped=None, + on_lifecycle_attempting_connect=None, + on_lifecycle_connection_success=None, + on_lifecycle_connection_failure=None, + on_lifecycle_disconnection=None): + proxy_options = self.get_proxy_options_for_mqtt_connection() + credentials_provider = auth.AwsCredentialsProvider.new_default_chain() + mqtt5_client = mqtt5_client_builder.websockets_with_default_aws_signing( + endpoint=self.get_command_required(self.m_cmd_endpoint), + region=self.get_command_required(self.m_cmd_signing_region), + credentials_provider=credentials_provider, + http_proxy_options=proxy_options, + ca_filepath=self.get_command(self.m_cmd_ca_file), + on_publish_received=on_publish_received, + on_lifecycle_stopped=on_lifecycle_stopped, + on_lifecycle_attempting_connect=on_lifecycle_attempting_connect, + on_lifecycle_connection_success=on_lifecycle_connection_success, + on_lifecycle_connection_failure=on_lifecycle_connection_failure, + on_lifecycle_disconnection=on_lifecycle_disconnection, + client_id=self.get_command_required("client_id")) + return mqtt5_client + + def build_direct_mqtt5_client(self, + on_publish_received=None, + on_lifecycle_stopped=None, + on_lifecycle_attempting_connect=None, + on_lifecycle_connection_success=None, + on_lifecycle_connection_failure=None, + on_lifecycle_disconnection=None): + proxy_options = self.get_proxy_options_for_mqtt_connection() + mqtt5_client = mqtt5_client_builder.mtls_from_path( + endpoint=self.get_command_required(self.m_cmd_endpoint), + port=self.get_command_required("port"), + cert_filepath=self.get_command_required(self.m_cmd_cert_file), + pri_key_filepath=self.get_command_required(self.m_cmd_key_file), + ca_filepath=self.get_command(self.m_cmd_ca_file), + http_proxy_options=proxy_options, + on_publish_received=on_publish_received, + on_lifecycle_stopped=on_lifecycle_stopped, + on_lifecycle_attempting_connect=on_lifecycle_attempting_connect, + on_lifecycle_connection_success=on_lifecycle_connection_success, + on_lifecycle_connection_failure=on_lifecycle_connection_failure, + on_lifecycle_disconnection=on_lifecycle_disconnection, + client_id=self.get_command_required("client_id")) + return mqtt5_client + + def build_mqtt5_client(self, + on_publish_received=None, + on_lifecycle_stopped=None, + on_lifecycle_attempting_connect=None, + on_lifecycle_connection_success=None, + on_lifecycle_connection_failure=None, + on_lifecycle_disconnection=None): + + if self.get_command(self.m_cmd_signing_region) is not None: + return self.build_websocket_mqtt5_client(on_publish_received=on_publish_received, + on_lifecycle_stopped=on_lifecycle_stopped, + on_lifecycle_attempting_connect=on_lifecycle_attempting_connect, + on_lifecycle_connection_success=on_lifecycle_connection_success, + on_lifecycle_connection_failure=on_lifecycle_connection_failure, + on_lifecycle_disconnection=on_lifecycle_disconnection) + else: + return self.build_direct_mqtt5_client(on_publish_received=on_publish_received, + on_lifecycle_stopped=on_lifecycle_stopped, + on_lifecycle_attempting_connect=on_lifecycle_attempting_connect, + on_lifecycle_connection_success=on_lifecycle_connection_success, + on_lifecycle_connection_failure=on_lifecycle_connection_failure, + on_lifecycle_disconnection=on_lifecycle_disconnection) # Constants for commonly used/needed commands m_cmd_endpoint = "endpoint" diff --git a/samples/mqtt5_pubsub.py b/samples/mqtt5_pubsub.py new file mode 100644 index 00000000..d8b5fe21 --- /dev/null +++ b/samples/mqtt5_pubsub.py @@ -0,0 +1,150 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0. + +import command_line_utils +from awscrt import mqtt5, exceptions +from uuid import uuid4 +import threading +from concurrent.futures import Future +import time + +TIMEOUT = 100 +topic_filter = "test/topic" + +# Parse arguments +cmdUtils = command_line_utils.CommandLineUtils("PubSub - Send and receive messages through an MQTT5 connection.") +cmdUtils.add_common_mqtt5_commands() +cmdUtils.add_common_topic_message_commands() +cmdUtils.add_common_proxy_commands() +cmdUtils.add_common_logging_commands() +cmdUtils.register_command("key", "", "Path to your key in PEM format.", True, str) +cmdUtils.register_command("cert", "", "Path to your client certificate in PEM format.", True, str) +cmdUtils.register_command( + "port", + "", + "Connection port. AWS IoT supports 433 and 8883 (optional, default=auto).", + type=int) +cmdUtils.register_command( + "client_id", + "", + "Client ID to use for MQTT5 connection (optional, default=None).", + default="test-" + str(uuid4())) +cmdUtils.register_command( + "count", + "", + "The number of messages to send (optional, default='10').", + default=10, + type=int) +# Needs to be called so the command utils parse the commands +cmdUtils.get_args() + +received_count = 0 +received_all_event = threading.Event() +future_stopped = Future() +future_connection_success = Future() + +# Callback when any publish is received + + +def on_publish_received(publish_packet): + assert isinstance(publish_packet, mqtt5.PublishPacket) + print("Received message from topic'{}':{}".format(publish_packet.topic, publish_packet.payload)) + global received_count + received_count += 1 + if received_count == cmdUtils.get_command("count"): + received_all_event.set() + + +# Callback for the lifecycle event Stopped +def on_lifecycle_stopped(lifecycle_stopped_data: mqtt5.LifecycleStoppedData): + print("Lifecycle Stopped") + global future_stopped + future_stopped.set_result(lifecycle_stopped_data) + + +# Callback for the lifecycle event Connection Success +def on_lifecycle_connection_success(lifecycle_connect_success_data: mqtt5.LifecycleConnectSuccessData): + print("Lifecycle Connection Success") + global future_connection_success + future_connection_success.set_result(lifecycle_connect_success_data) + + +# Callback for the lifecycle event Connection Failure +def on_lifecycle_connection_failure(lifecycle_connection_failure: mqtt5.LifecycleConnectFailureData): + print("Lifecycle Connection Failure") + print("Connection failed with error_code:{}".format(exceptions.from_code(lifecycle_connection_failure.error_code))) + + +if __name__ == '__main__': + print("\nStarting MQTT5 PubSub Sample\n") + message_count = cmdUtils.get_command("count") + message_topic = cmdUtils.get_command(cmdUtils.m_cmd_topic) + message_string = cmdUtils.get_command(cmdUtils.m_cmd_message) + + client = cmdUtils.build_mqtt5_client( + on_publish_received=on_publish_received, + on_lifecycle_stopped=on_lifecycle_stopped, + on_lifecycle_connection_success=on_lifecycle_connection_success, + on_lifecycle_connection_failure=on_lifecycle_connection_failure) + print("MQTT5 Client Created") + + client.start() + lifecycle_connect_success_data = future_connection_success.result(TIMEOUT) + connack_packet = lifecycle_connect_success_data.connack_packet + negotiated_settings = lifecycle_connect_success_data.negotiated_settings + print("Connected to endpoint:'{}' with Client ID:'{}' with reason_code:{}".format( + cmdUtils.get_command(cmdUtils.m_cmd_endpoint), + connack_packet.assigned_client_identifier, + repr(connack_packet.reason_code))) + + # Subscribe + + print("Subscribing to topic '{}'...".format(message_topic)) + subscribe_future = client.subscribe(subscribe_packet=mqtt5.SubscribePacket( + subscriptions=[mqtt5.Subscription( + topic_filter=message_topic, + qos=mqtt5.QoS.AT_LEAST_ONCE)] + )) + suback = subscribe_future.result(TIMEOUT) + print("Subscribed with {}".format(suback.reason_codes)) + + # Publish message to server desired number of times. + # This step is skipped if message is blank. + # This step loops forever if count was set to 0. + if message_string: + if message_count == 0: + print("Sending messages until program killed") + else: + print("Sending {} message(s)".format(message_count)) + + publish_count = 1 + while (publish_count <= message_count) or (message_count == 0): + message = "{} [{}]".format(message_string, publish_count) + print("Publishing message to topic '{}': {}".format(message_topic, message)) + publish_future = client.publish(mqtt5.PublishPacket( + topic=message_topic, + payload=message_string, + qos=mqtt5.QoS.AT_LEAST_ONCE + )) + + publish_completion_data = publish_future.result(TIMEOUT) + print("PubAck received with {}".format(repr(publish_completion_data.puback.reason_code))) + time.sleep(1) + publish_count += 1 + + received_all_event.wait(TIMEOUT) + print("{} message(s) received.".format(received_count)) + + # Unsubscribe + + print("Unsubscribing from topic '{}'".format(message_topic)) + unsubscribe_future = client.unsubscribe(unsubscribe_packet=mqtt5.UnsubscribePacket( + topic_filters=[message_topic])) + unsuback = unsubscribe_future.result(TIMEOUT) + print("Unsubscribed with {}".format(unsuback.reason_codes)) + + print("Stopping Client") + client.stop() + + future_stopped.result(TIMEOUT) + print("Client Stopped!") From 5f4f2dbec5b76935a9dc736e5cec3f5c4b6076ca Mon Sep 17 00:00:00 2001 From: Steve Kim <86316075+sbSteveK@users.noreply.github.com> Date: Thu, 3 Nov 2022 14:09:47 -0700 Subject: [PATCH 02/61] sample instructions for mqtt5-pubsub (#15) sample instructions for mqtt5-pubsub --- samples/README.md | 60 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/samples/README.md b/samples/README.md index 8c4748a5..765d18f0 100644 --- a/samples/README.md +++ b/samples/README.md @@ -1,5 +1,6 @@ # Sample apps for the AWS IoT Device SDK v2 for Python +* [MQTT5 PubSub](#mqtt5-pubsub) * [PubSub](#pubsub) * [Basic Connect](#basic-connect) * [Websocket Connect](#websocket-connect) @@ -22,6 +23,65 @@ Then change into the samples directory to run the Python commands to execute the python3 pubsub.py --help ``` +## MQTT5 PubSub +This sample uses the +[Message Broker](https://docs.aws.amazon.com/iot/latest/developerguide/iot-message-broker.html) +for AWS IoT to send and receive messages +through an MQTT5 connection. On startup, the device connects to the server, +subscribes to a topic, and begins publishing messages to that topic. +The device should receive those same messages back from the message broker, +since it is subscribed to that same topic. +Status updates are continually printed to the console. + +Source: `samples/mqtt5_pubsub.py` + +Your Thing's [Policy](https://docs.aws.amazon.com/iot/latest/developerguide/iot-policies.html) must provide privileges for this sample to connect, subscribe, publish, and receive. Make sure your policy allows a client ID of `test-*` to connect or use `--client_id ` to send the client ID your policy supports. + +
+(see sample policy) +
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Action": [
+        "iot:Publish",
+        "iot:Receive"
+      ],
+      "Resource": [
+        "arn:aws:iot:region:account:topic/test/topic"
+      ]
+    },
+    {
+      "Effect": "Allow",
+      "Action": [
+        "iot:Subscribe"
+      ],
+      "Resource": [
+        "arn:aws:iot:region:account:topicfilter/test/topic"
+      ]
+    },
+    {
+      "Effect": "Allow",
+      "Action": [
+        "iot:Connect"
+      ],
+      "Resource": [
+        "arn:aws:iot:region:account:client/test-*"
+      ]
+    }
+  ]
+}
+
+
+ +Run the sample like this: +``` sh +# For Windows: replace 'python3' with 'python' +python3 mqtt5_pubsub.py --endpoint --ca_file --cert --key +``` + ## PubSub This sample uses the From fb665ffb86481e1637934ba3e8e501a3d73cba9c Mon Sep 17 00:00:00 2001 From: Steve Kim Date: Thu, 3 Nov 2022 14:41:50 -0700 Subject: [PATCH 03/61] error_code fix in mqtt5_pubsub sample --- samples/mqtt5_pubsub.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/mqtt5_pubsub.py b/samples/mqtt5_pubsub.py index d8b5fe21..ba62e1ea 100644 --- a/samples/mqtt5_pubsub.py +++ b/samples/mqtt5_pubsub.py @@ -72,7 +72,7 @@ def on_lifecycle_connection_success(lifecycle_connect_success_data: mqtt5.Lifecy # Callback for the lifecycle event Connection Failure def on_lifecycle_connection_failure(lifecycle_connection_failure: mqtt5.LifecycleConnectFailureData): print("Lifecycle Connection Failure") - print("Connection failed with error_code:{}".format(exceptions.from_code(lifecycle_connection_failure.error_code))) + print("Connection failed with error_code:{}".format(lifecycle_connection_failure.exception)) if __name__ == '__main__': From f566ea0cbeaa5575ba9feebe477da68d54836444 Mon Sep 17 00:00:00 2001 From: Steve Kim Date: Thu, 3 Nov 2022 14:43:23 -0700 Subject: [PATCH 04/61] wording change in sample log --- samples/mqtt5_pubsub.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/mqtt5_pubsub.py b/samples/mqtt5_pubsub.py index ba62e1ea..cc8ca230 100644 --- a/samples/mqtt5_pubsub.py +++ b/samples/mqtt5_pubsub.py @@ -72,7 +72,7 @@ def on_lifecycle_connection_success(lifecycle_connect_success_data: mqtt5.Lifecy # Callback for the lifecycle event Connection Failure def on_lifecycle_connection_failure(lifecycle_connection_failure: mqtt5.LifecycleConnectFailureData): print("Lifecycle Connection Failure") - print("Connection failed with error_code:{}".format(lifecycle_connection_failure.exception)) + print("Connection failed with exception:{}".format(lifecycle_connection_failure.exception)) if __name__ == '__main__': From a16b920c618a14a67320e27194b6edfc1aeb412e Mon Sep 17 00:00:00 2001 From: Steve Kim Date: Mon, 7 Nov 2022 16:21:09 -0800 Subject: [PATCH 05/61] first draft MQTT5 usage guide --- MQTT5.md | 204 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 204 insertions(+) create mode 100644 MQTT5.md diff --git a/MQTT5.md b/MQTT5.md new file mode 100644 index 00000000..6b3ee91d --- /dev/null +++ b/MQTT5.md @@ -0,0 +1,204 @@ +# MQTT 5 +## Developer Preview Disclaimer +MQTT5 support is currently in **developer preview**. We encourage feedback at all times, but feedback during the preview window is especially valuable in shaping the final product. During the preview period we may make backwards-incompatible changes to the public API, but in general, this is something we will try our best to avoid. + +The MQTT5 client cannot yet be used with the AWS IoT MQTT services (Shadow, Jobs, Identity). This is a shortcoming that we hope to address in the near future. + +## What's Different? (relative to the MQTT311 implementation) +SDK MQTT5 support comes from a separate client implementation. In doing so, we took the opportunity to incorporate feedback about the 311 client that we could not apply without making breaking changes. If you're used to the 311 client's API contract, there are a number of differences. + +### Major changes +* The MQTT5 client does not treat initial connection failures differently. With the 311 implementation, a failure during initial connect would halt reconnects completely. +* The set of client lifecycle events is expanded and contains more detailed information whenever possible. All protocol data is exposed to the user. +* MQTT operations are completed with the full associated ACK packet when possible. +* New behavioral configuration options: + * IoT Core specific validation - will validate and fail operations that break IoT Core specific restrictions + * IoT Core specific flow control - will apply flow control to honor IoT Core specific per-connection limits and quotas + * Flexible queue control - provides a number of options to control what happens to incomplete operations on a disconnection event +* A new API has been added to query the internal state of the client's operation queue. This API allows the user to make more informed flow control decisions before submitting operatons to the client. +* Data can no longer back up on the socket. At most one frame of data is ever pending-write on the socket. +* The MQTT5 client has a single message-received callback. Per-subscription callbacks are not supported. + +### Minor changes +* Public API terminology has changed. You *start* or *stop* the MQTT5 client. This removes the semantic confusion with connect/disconnect as client-level controls vs. internal recurrent networking events. +* With the 311 implementation, there were two separate objects, a client and a connection. With MQTT5, there is only the client. + +### Not Supported +Not all parts of the MQTT5 spec are supported by the implementation. We currently do not support: +* AUTH packets and the authentication fields in the CONNECT packet +* QoS 2 + +## Client lifecycle management +Once created, an MQTT5 client's configuration is immutable. Invoking start() on the client will put it into an active state where it +recurrently establishes a connection to the configured remote endpoint. Reconnecting continues until you invoke stop(). + +``` +# Create an MQTT5 Client + + client_options = mqtt5.ClientOptions( + host_name = , + port = ) + + # Other options in client options can be set but once Client is initialized configuration is immutable + # e.g. setting the on_publish_callback_fn to be called + # client_options.on_publish_callback_fn = on_publish_received + + client = mqtt5.Client(client_options) + + +# Use the client + client.start(); + ... +``` + +Invoking stop() breaks the current connection (if any) and moves the client into an idle state. + +``` + // Shutdown + client.stop(); + +``` + +## Connecting To AWS IoT Core +We strongly recommend using the AwsIotMqtt5ClientConfigBuilder class to configure MQTT5 clients when connecting to AWS IoT Core. The builder simplifies configuration for all authentication methods supported by AWS IoT Core. This section shows samples for all of the authentication possibilities. + +#### **Direct MQTT with X509-based mutual TLS** +For X509 based mutual TLS, you can create a client where the certificate and private key are configured by path: +``` + # X.509 based certificate file + certificate_file_path = "" + # PKCS#1 or PKCS#8 PEM encoded private key file + private_key_filePath = "" + + # other builder configurations can be added using **kwargs in the builder + + # Create an MQTT5 Client using mqtt5_client_builder + client = mqtt5_client_builder.mtls_from_path( + endpoint = , + cert_filepath=certificate_file_path, + pri_key_filepath=private_key_filePath)) +``` + +#### **MQTT over Websockets with Sigv4 authentication** +Sigv4-based authentication requires a credentials provider capable of sourcing valid AWS credentials. Sourced credentials +will sign the websocket upgrade request made by the client while connecting. The default credentials provider chain supported by +the SDK is capable of resolving credentials in a variety of environments according to a chain of priorities: + +```Environment -> Profile (local file system) -> STS Web Identity -> IMDS (ec2) or ECS``` +If the default credentials provider chain and built-in AWS region extraction logic are sufficient, you do not need to specify +any additional configuration: +``` + # The signing region. e.x.: 'us-east-1' + signing_region = + credentials_provider = auth.AwsCredentialsProvider.new_dfault_chain() + + # other builder configurations can be added using **kwargs in the builder + + # Create an MQTT5 Client using mqtt5_client_builder + client = mqtt5_client_builder.websockets_with_default_aws_signing( + endpoint = , + region = signing_region, + credentials_provider=credentials_provider)) +``` + +#### **Direct MQTT with Custom Authentication** +AWS IoT Core Custom Authentication allows you to use a lambda to gate access to IoT Core resources. For this authentication method, +you must supply an additional configuration structure containing fields relevant to AWS IoT Core Custom Authentication. +If your custom authenticator does not use signing, you don't specify anything related to the token signature: +``` + # other builder configurations can be added using **kwargs in the builder + + client = mqtt5_client_builder.direct_with_custom_authorizer( + endpoint = , + auth_authorizer_name = , + auth_username = , + auth_password =) +``` +If your custom authorizer uses signing, you must specify the three signed token properties as well. The token signature must be the URI-encoding of the base64 encoding of the digital signature of the token value via the private key associated with the public key that was registered with the custom authorizer. It is your responsibility to URI-encode the token signature. +``` + # other builder configurations can be added using **kwargs in the builder + + client = mqtt5_client_builder.direct_with_custom_authorizer( + endpoint = , + auth_authorizer_name = , + auth_username = , + auth_password =, + auth_authorizer_signature=) +``` +In both cases, the builder will construct a final CONNECT packet username field value for you based on the values configured. Do not add the token-signing fields to the value of the username that you assign within the custom authentication config structure. Similarly, do not add any custom authentication related values to the username in the CONNECT configuration optionally attached to the client configuration. The builder will do everything for you. + +#### **HTTP Proxy** +No matter what your connection transport or authentication method is, you may connect through an HTTP proxy +by adding the http_proxy_options keyword argument to the builder: +``` + http_proxy_options = http.HttpProxyOptions( + host_name = , + port = ) + + # Create an MQTT5 Client using mqtt5_client_builder with proxy options as keyword argument + client = mqtt5_client_builder.mtls_from_path( + endpoint = , + cert_filepath = , + pri_key_filepath = , + http_proxy_options = http_proxy_options)) +``` +SDK Proxy support also includes support for basic authentication and TLS-to-proxy. SDK proxy support does not include any additional +proxy authentication methods (kerberos, NTLM, etc...) nor does it include non-HTTP proxies (SOCKS5, for example). + +## Lifecycle Events +The MQTT5 client emits a set of events related to state and network status changes. + +#### **AttemptingConnect** +Emitted when the client begins to make a connection attempt. + +#### **ConnectionSuccess** +Emitted when a connection attempt succeeds based on receipt of an affirmative CONNACK packet from the MQTT broker. A ConnectionSuccess event includes the MQTT broker's CONNACK packet, as well as a structure -- the NegotiatedSettings -- which contains the final values for all variable MQTT session settings (based on protocol defaults, client wishes, and server response). + +#### **ConnectionFailure** +Emitted when a connection attempt fails at any point between DNS resolution and CONNACK receipt. In addition to an error code, additional data may be present in the event based on the context. For example, if the remote endpoint sent a CONNACK with a failing reason code, the CONNACK packet will be included in the event data. + +#### **Disconnect** +Emitted when the client's network connection is shut down, either by a local action, event, or a remote close or reset. Only emitted after a ConnectionSuccess event: a network connection that is shut down during the connecting process manifests as a ConnectionFailure event. A Disconnect event will always include an error code. If the Disconnect event is due to the receipt of a server-sent DISCONNECT packet, the packet will be included with the event data. + +#### **Stopped** +Emitted once the client has shutdown any associated network connection and entered an idle state where it will no longer attempt to reconnect. Only emitted after an invocation of stop() on the client. A stopped client may always be started again. + +## Client Operations +There are four basic MQTT operations you can perform with the MQTT5 client. + +### Subscribe +The Subscribe operation takes a description of the SUBSCRIBE packet you wish to send and returns a future that resolves successfully with the corresponding SUBACK returned by the broker; the future result raises an exception if anything goes wrong before the SUBACK is received. +``` + subscribe_future = client.subscribe(subscribe_packet = mqtt5.SubscribePacket( + subscriptions = [mqtt5.Subscription( + topic_filter = "hello/world/qos1", + qos = mqtt5.QoS.AT_LEAST_ONCE)])) + + suback = subscribe_future.result() +``` +### Unsubscribe +The Unsubscribe operation takes a description of the UNSUBSCRIBE packet you wish to send and returns a future that resolves successfully with the corresponding UNSUBACK returned by the broker; the future result raises an exception if anything goes wrong before the UNSUBACK is received. +``` + unsubscribe_future = client.unsubscribe(unsubscribe_packet = mqtt5.UnsubscribePacket( + topic_filters=["hello/world/qos1"])) + + unsuback = unsubscribe_future.result() +``` +### Publish +The Publish operation takes a description of the PUBLISH packet you wish to send and returns a future of polymorphic value. If the PUBLISH was a QoS 0 publish, then the future result is an empty PUBACK packet with all members set to None and is completed as soon as the packet has been written to the socket. If the PUBLISH was a QoS 1 publish, then the future result is a PUBACK packet value and is completed as soon as the PUBACK is received from the broker. If the operation fails for any reason before these respective completion events, the future result raises an exception. +``` + publish_future = client.publish(mqtt5.PublishPacket( + topic = "hello/world/qos1", + payload = "This is the payload of a QoS 1 publish", + qos = mqtt5.QoS.AT_LEAST_ONCE)) + + # on success, the result of publish_future will be a PubackPacket + puback = publish_future.result() +``` +### Disconnect +The stop() API supports a DISCONNECT packet as an optional parameter. If supplied, the DISCONNECT packet will be sent to the server prior to closing the socket. There is no future returned by a call to stop() but you may listen for the 'stopped' event on the client. +``` + client.stop(mqtt5.DisconnectPacket( + reason_code = mqtt5.DisconnectReasonCode.NORMAL_DISCONNECTION, + session_expiry_interval_sec = 3600)) +``` \ No newline at end of file From b6688e496d2669ec46d67d61b74912771ebe4f02 Mon Sep 17 00:00:00 2001 From: Steve Kim Date: Wed, 30 Nov 2022 15:26:39 -0800 Subject: [PATCH 06/61] changed awscrt version to 0.16.0 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index bc834fa8..f52a6a12 100644 --- a/setup.py +++ b/setup.py @@ -40,7 +40,7 @@ def _load_version(): "Operating System :: OS Independent", ], install_requires=[ - 'awscrt==0.15.2', + 'awscrt==0.16.0', ], python_requires='>=3.7', ) From 67ad576b73f74edfe71cb9ee01249fd99ad0549a Mon Sep 17 00:00:00 2001 From: Steve Kim Date: Wed, 30 Nov 2022 15:57:40 -0800 Subject: [PATCH 07/61] MQTT5 document moved --- docsrc/conf.py | 2 +- MQTT5.md => documents/MQTT5.md | 24 +++++++++++++++++++----- 2 files changed, 20 insertions(+), 6 deletions(-) rename MQTT5.md => documents/MQTT5.md (85%) diff --git a/docsrc/conf.py b/docsrc/conf.py index 01eb46f6..2e66aa2c 100644 --- a/docsrc/conf.py +++ b/docsrc/conf.py @@ -33,7 +33,7 @@ 'sphinx.ext.autodoc', 'sphinx.ext.autodoc.typehints', 'sphinx.ext.napoleon', - 'sphinx.ext.intersphinx', # for linking external docs (ex: aws-crt-python) + 'sphinx.ext.intersphinx', # for linking external docs (ex: aws-crt-python) ] # Add any paths that contain templates here, relative to this directory. diff --git a/MQTT5.md b/documents/MQTT5.md similarity index 85% rename from MQTT5.md rename to documents/MQTT5.md index 6b3ee91d..5fab7739 100644 --- a/MQTT5.md +++ b/documents/MQTT5.md @@ -4,6 +4,20 @@ MQTT5 support is currently in **developer preview**. We encourage feedback at a The MQTT5 client cannot yet be used with the AWS IoT MQTT services (Shadow, Jobs, Identity). This is a shortcoming that we hope to address in the near future. +## Introduction + +This user guide is designed to act as a reference and guide for how to use MQTT5 with the Java SDK. This guide includes code snippets for how to make a MQTT5 client with proper configuration, how to connect to AWS IoT Core, how to perform operations and interact with AWS IoT Core through MQTT5, and some best practices for MQTT5. + +If you are completely new to MQTT, it is highly recommended to check out the following resources to learn more about MQTT: + +* MQTT.org getting started: https://mqtt.org/getting-started/ +* MQTT.org FAQ (includes list of commonly used terms): https://mqtt.org/faq/ +* MQTT on AWS IoT Core documentation: https://docs.aws.amazon.com/iot/latest/developerguide/mqtt.html +* MQTT 5 standard: https://docs.oasis-open.org/mqtt/mqtt/v5.0/os/mqtt-v5.0-os.html +* MQTT 311 standard: https://docs.oasis-open.org/mqtt/mqtt/v3.1.1/os/mqtt-v3.1.1-os.html + +This user guide expects some beginner level familiarity with MQTT and the terms used to describe MQTT. + ## What's Different? (relative to the MQTT311 implementation) SDK MQTT5 support comes from a separate client implementation. In doing so, we took the opportunity to incorporate feedback about the 311 client that we could not apply without making breaking changes. If you're used to the 311 client's API contract, there are a number of differences. @@ -12,15 +26,15 @@ SDK MQTT5 support comes from a separate client implementation. In doing so, we * The set of client lifecycle events is expanded and contains more detailed information whenever possible. All protocol data is exposed to the user. * MQTT operations are completed with the full associated ACK packet when possible. * New behavioral configuration options: - * IoT Core specific validation - will validate and fail operations that break IoT Core specific restrictions - * IoT Core specific flow control - will apply flow control to honor IoT Core specific per-connection limits and quotas - * Flexible queue control - provides a number of options to control what happens to incomplete operations on a disconnection event -* A new API has been added to query the internal state of the client's operation queue. This API allows the user to make more informed flow control decisions before submitting operatons to the client. + * [IoT Core specific validation](https://awslabs.github.io/aws-crt-python/api/mqtt5.html#awscrt.mqtt5.ExtendedValidationAndFlowControlOptions) - will validate and fail operations that break IoT Core specific restrictions + * [IoT Core specific flow control](https://awslabs.github.io/aws-crt-python/api/mqtt5.html#awscrt.mqtt5.ExtendedValidationAndFlowControlOptions) - will apply flow control to honor IoT Core specific per-connection limits and quotas + * [Flexible queue control](https://awslabs.github.io/aws-crt-python/api/mqtt5.html#awscrt.mqtt5.ClientOperationQueueBehaviorType) - provides a number of options to control what happens to incomplete operations on a disconnection event +* A [new API](https://awslabs.github.io/aws-crt-python/api/mqtt5.html#awscrt.mqtt5.Client) has been added to query the internal state of the client's operation queue. This API allows the user to make more informed flow control decisions before submitting operatons to the client. * Data can no longer back up on the socket. At most one frame of data is ever pending-write on the socket. * The MQTT5 client has a single message-received callback. Per-subscription callbacks are not supported. ### Minor changes -* Public API terminology has changed. You *start* or *stop* the MQTT5 client. This removes the semantic confusion with connect/disconnect as client-level controls vs. internal recurrent networking events. +* Public API terminology has changed. You `start()` or `stop()` the MQTT5 client. This removes the semantic confusion with connect/disconnect as client-level controls vs. internal recurrent networking events. * With the 311 implementation, there were two separate objects, a client and a connection. With MQTT5, there is only the client. ### Not Supported From cc038f77ac3852ca4fca4f62702ce495cde446cb Mon Sep 17 00:00:00 2001 From: Steve Kim Date: Wed, 30 Nov 2022 16:31:21 -0800 Subject: [PATCH 08/61] Added mqtt5 pubsub sample to codebuild --- codebuild/samples/linux-smoke-tests.yml | 1 + codebuild/samples/pubsub-mqtt5-linux.sh | 15 +++++++++++++++ 2 files changed, 16 insertions(+) create mode 100644 codebuild/samples/pubsub-mqtt5-linux.sh diff --git a/codebuild/samples/linux-smoke-tests.yml b/codebuild/samples/linux-smoke-tests.yml index 0c931f61..e2faa151 100644 --- a/codebuild/samples/linux-smoke-tests.yml +++ b/codebuild/samples/linux-smoke-tests.yml @@ -19,6 +19,7 @@ phases: - $CODEBUILD_SRC_DIR/codebuild/samples/connect-linux.sh - $CODEBUILD_SRC_DIR/codebuild/samples/custom-auth-linux.sh - $CODEBUILD_SRC_DIR/codebuild/samples/pkcs11-connect-linux.sh + - $CODEBUILD_SRC_DIR/codebuild/samples/pubsub-mqtt5-linux.sh - $CODEBUILD_SRC_DIR/codebuild/samples/pubsub-linux.sh - $CODEBUILD_SRC_DIR/codebuild/samples/shadow-linux.sh post_build: diff --git a/codebuild/samples/pubsub-mqtt5-linux.sh b/codebuild/samples/pubsub-mqtt5-linux.sh new file mode 100644 index 00000000..ee0e76b0 --- /dev/null +++ b/codebuild/samples/pubsub-mqtt5-linux.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +set -e +set -o pipefail + +env + +pushd $CODEBUILD_SRC_DIR/samples/ + +ENDPOINT=$(aws secretsmanager get-secret-value --secret-id "ci/endpoint" --query "SecretString" | cut -f2 -d":" | sed -e 's/[\\\"\}]//g') + +echo "MQTT5 PubSub test" +python3 mqtt5_pubsub.py --endpoint $ENDPOINT --key /tmp/privatekey.pem --cert /tmp/certificate.pem + +popd \ No newline at end of file From 8a3ff0e2fc96a1bb1cc596528c1bbbdc7c633848 Mon Sep 17 00:00:00 2001 From: Steve Kim Date: Wed, 30 Nov 2022 16:39:01 -0800 Subject: [PATCH 09/61] test --- codebuild/samples/pubsub-mqtt5-linux.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codebuild/samples/pubsub-mqtt5-linux.sh b/codebuild/samples/pubsub-mqtt5-linux.sh index ee0e76b0..689746ca 100644 --- a/codebuild/samples/pubsub-mqtt5-linux.sh +++ b/codebuild/samples/pubsub-mqtt5-linux.sh @@ -10,6 +10,6 @@ pushd $CODEBUILD_SRC_DIR/samples/ ENDPOINT=$(aws secretsmanager get-secret-value --secret-id "ci/endpoint" --query "SecretString" | cut -f2 -d":" | sed -e 's/[\\\"\}]//g') echo "MQTT5 PubSub test" -python3 mqtt5_pubsub.py --endpoint $ENDPOINT --key /tmp/privatekey.pem --cert /tmp/certificate.pem +python3 mqtt5_pubsub.py --endpoint $ENDPOINT --key /tmp/privatekey.pem --cert /tmp/certificate.pem --verbosity Trace popd \ No newline at end of file From c8313a31a92d8cbd22cb16894298f60f3bc15e31 Mon Sep 17 00:00:00 2001 From: Steve Kim Date: Wed, 30 Nov 2022 16:48:12 -0800 Subject: [PATCH 10/61] chmod on pubsub script --- codebuild/samples/pubsub-mqtt5-linux.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 codebuild/samples/pubsub-mqtt5-linux.sh diff --git a/codebuild/samples/pubsub-mqtt5-linux.sh b/codebuild/samples/pubsub-mqtt5-linux.sh old mode 100644 new mode 100755 From 2d71cb6692001b8d2da1a119e443be10732e18ce Mon Sep 17 00:00:00 2001 From: Steve Kim Date: Wed, 30 Nov 2022 17:02:43 -0800 Subject: [PATCH 11/61] ack_timeout_sec --- awsiot/mqtt5_client_builder.py | 6 +++--- codebuild/samples/pubsub-mqtt5-linux.sh | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/awsiot/mqtt5_client_builder.py b/awsiot/mqtt5_client_builder.py index 973bf562..e7073b4c 100644 --- a/awsiot/mqtt5_client_builder.py +++ b/awsiot/mqtt5_client_builder.py @@ -106,7 +106,7 @@ **connack_timeout_ms** (`int`): The time interval to wait after sending a CONNECT request for a CONNACK to arrive. If one does not arrive, the connection will be shut down. - **operation_timeout_sec** (`int`): The time interval to wait for an ack after sending a QoS 1+ PUBLISH, SUBSCRIBE, + **ack_timeout_sec** (`int`): The time interval to wait for an ack after sending a QoS 1+ PUBLISH, SUBSCRIBE, or UNSUBSCRIBE before failing the operation. **on_publish_received** (`Callable`): Callback invoked for all publish packets received by client. @@ -281,8 +281,8 @@ def _builder( client_options.ping_timeout_ms = _get(kwargs, 'ping_timeout_ms') if client_options.connack_timeout_ms is None: client_options.connack_timeout_ms = _get(kwargs, 'connack_timeout_ms') - if client_options.operation_timeout_sec is None: - client_options.operation_timeout_sec = _get(kwargs, 'operation_timeout_sec') + if client_options.ack_timeout_sec is None: + client_options.ack_timeout_sec = _get(kwargs, 'ack_timeout_sec') if client_options.websocket_handshake_transform is None: client_options.websocket_handshake_transform = websocket_handshake_transform diff --git a/codebuild/samples/pubsub-mqtt5-linux.sh b/codebuild/samples/pubsub-mqtt5-linux.sh index 689746ca..ee0e76b0 100755 --- a/codebuild/samples/pubsub-mqtt5-linux.sh +++ b/codebuild/samples/pubsub-mqtt5-linux.sh @@ -10,6 +10,6 @@ pushd $CODEBUILD_SRC_DIR/samples/ ENDPOINT=$(aws secretsmanager get-secret-value --secret-id "ci/endpoint" --query "SecretString" | cut -f2 -d":" | sed -e 's/[\\\"\}]//g') echo "MQTT5 PubSub test" -python3 mqtt5_pubsub.py --endpoint $ENDPOINT --key /tmp/privatekey.pem --cert /tmp/certificate.pem --verbosity Trace +python3 mqtt5_pubsub.py --endpoint $ENDPOINT --key /tmp/privatekey.pem --cert /tmp/certificate.pem popd \ No newline at end of file From 8eeb9651522e50950923faa85cda8fda3c8bdc5f Mon Sep 17 00:00:00 2001 From: Steve Kim Date: Wed, 30 Nov 2022 17:06:19 -0800 Subject: [PATCH 12/61] publish_packet_data --- samples/mqtt5_pubsub.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/samples/mqtt5_pubsub.py b/samples/mqtt5_pubsub.py index cc8ca230..b171cb7e 100644 --- a/samples/mqtt5_pubsub.py +++ b/samples/mqtt5_pubsub.py @@ -46,7 +46,8 @@ # Callback when any publish is received -def on_publish_received(publish_packet): +def on_publish_received(publish_packet_data): + publish_packet = publish_packet_data.publish_packet assert isinstance(publish_packet, mqtt5.PublishPacket) print("Received message from topic'{}':{}".format(publish_packet.topic, publish_packet.payload)) global received_count From 00ea8a397c0d81ca27a8109bb445d24ab376df7d Mon Sep 17 00:00:00 2001 From: Steve Kim Date: Thu, 1 Dec 2022 09:22:47 -0800 Subject: [PATCH 13/61] run mqtt5 sample in all ci jobs --- .github/workflows/ci.yml | 28 +++++++- utils/mqtt5_test_setup.sh | 144 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 171 insertions(+), 1 deletion(-) create mode 100755 utils/mqtt5_test_setup.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 65c02f94..4c713db1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,6 +27,7 @@ env: CI_JOBS_ROLE: ${{ secrets.AWS_CI_JOBS_ROLE }} CI_FLEET_PROVISIONING_ROLE: ${{ secrets.AWS_CI_FLEET_PROVISIONING_ROLE }} CI_DEVICE_ADVISOR: ${{ secrets.AWS_CI_DEVICE_ADVISOR_ROLE }} + CI_MQTT5_ROLE: ${{ secrets.AWS_CI_MQTT5_ROLE }} jobs: all-python-versions: @@ -96,6 +97,19 @@ jobs: run: | python -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder.pyz')" python builder.pyz build -p ${{ env.PACKAGE_NAME }} + + - name: configure AWS credentials (MQTT5) + uses: aws-actions/configure-aws-credentials@v1 + with: + role-to-assume: ${{ env.CI_MQTT5_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} + # - name: MQTT5 tests + # shell: bash + # run: | + # source utils/mqtt5_test_setup.sh s3://iot-sdk-ci-bucket-us-east1/IotUsProdMqtt5EnvironmentVariables.txt us-east-1 + # mvn test -Dtest=Mqtt5BuilderTest -DfailIfNoTests=false + # source utils/mqtt5_test_setup.sh s3://iot-sdk-ci-bucket-us-east1/IotUsProdMqtt5EnvironmentVariables.txt cleanup + - name: Running samples in CI setup run: | python -m pip install boto3 @@ -107,6 +121,9 @@ jobs: - name: run PubSub sample run: | python ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --language Python --sample_file "${{ env.CI_SAMPLES_FOLDER }}/pubsub.py" --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/PubSub/cert' --sample_secret_private_key 'ci/PubSub/key' + - name: run MQTT5 PubSub sample + run: | + python ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --language Python --sample_file "${{ env.CI_SAMPLES_FOLDER }}/mqtt5_pubsub.py" --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/PubSub/cert' --sample_secret_private_key 'ci/PubSub/key' - name: run Windows Certificate Connect sample run: | python ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --language Python --sample_file "${{ env.CI_SAMPLES_FOLDER }}/windows_cert_connect.py" --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/PubSub/cert' --sample_secret_private_key 'ci/PubSub/key' --sample_run_certutil true @@ -141,6 +158,9 @@ jobs: - name: run PubSub sample run: | python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --language Python --sample_file "${{ env.CI_SAMPLES_FOLDER }}/pubsub.py" --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/PubSub/cert' --sample_secret_private_key 'ci/PubSub/key' + - name: run MQTT5 PubSub sample + run: | + python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --language Python --sample_file "${{ env.CI_SAMPLES_FOLDER }}/mqtt5_pubsub.py" --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/PubSub/cert' --sample_secret_private_key 'ci/PubSub/key' - name: configure AWS credentials (Device Advisor) uses: aws-actions/configure-aws-credentials@v1 with: @@ -172,6 +192,9 @@ jobs: - name: run PubSub sample run: | python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --language Python --sample_file "${{ env.CI_SAMPLES_FOLDER }}/pubsub.py" --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/PubSub/cert' --sample_secret_private_key 'ci/PubSub/key' + - name: run MQTT5 PubSub sample + run: | + python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --language Python --sample_file "${{ env.CI_SAMPLES_FOLDER }}/mqtt5_pubsub.py" --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/PubSub/cert' --sample_secret_private_key 'ci/PubSub/key' - name: configure AWS credentials (Device Advisor) uses: aws-actions/configure-aws-credentials@v1 with: @@ -213,7 +236,10 @@ jobs: - name: run PubSub sample run: | python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --language Python --sample_file "${{ env.CI_SAMPLES_FOLDER }}/pubsub.py" --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/PubSub/cert' --sample_secret_private_key 'ci/PubSub/key' - - name: run PKCS11 Connect sample + - name: run MQTT5 PubSub sample + run: | + python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --language Python --sample_file "${{ env.CI_SAMPLES_FOLDER }}/mqtt5_pubsub.py" --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/PubSub/cert' --sample_secret_private_key 'ci/PubSub/key' + - name: run PKCS11 Connect sample run: | mkdir -p /tmp/tokens export SOFTHSM2_CONF=/tmp/softhsm2.conf diff --git a/utils/mqtt5_test_setup.sh b/utils/mqtt5_test_setup.sh new file mode 100755 index 00000000..390b7fe5 --- /dev/null +++ b/utils/mqtt5_test_setup.sh @@ -0,0 +1,144 @@ +#!/bin/bash + +# Get the S3 URL containing all of the MQTT5 testing environment variables passed in to the bash script +testing_env_bucket=$1 +region=$2 + +# Make sure we have something: +if [ "${testing_env_bucket}" != "" ] && [ "${region}" != "" ]; then + echo "S3 bucket for environment variables found and region" +else + echo "Could not get S3 bucket for environment variables and/or region." + echo "You need to run this script and pass the S3 URL of the file containing" + echo "all of the environment variables to set, as well as the secrets for certificates and private keys" + echo "" + echo "Example: mqtt5_test_setup.sh s3:/// " + echo "" + echo "When finished, run 'cleanup' to remove the files downloaded:" + echo "" + echo "Example: mqtt5_test_setup.sh s3:/// cleanup" + echo "" + return 1 +fi + +# Is this just a request to clean up? +# NOTE: This blindly assumes there is a environment_files.txt file +if [ "${region}" != "cleanup" ]; then + sleep 0.1 # we have to do something to do an else... +else + echo "Undoing environment variables" + unset $(grep -v '^#' ${PWD}/environment_files.txt | xargs | cut -d "=" -f 1) + unset AWS_TEST_MQTT5_CERTIFICATE_FILE + unset AWS_TEST_MQTT5_KEY_FILE + unset AWS_TEST_MQTT5_IOT_CERTIFICATE_PATH + unset AWS_TEST_MQTT5_IOT_KEY_PATH + + echo "Cleaning up resources..." + rm "${PWD}/environment_files.txt" + rm "${PWD}/crt_certificate.pem" + rm "${PWD}/crt_privatekey.pem" + rm "${PWD}/iot_certificate.pem" + rm "${PWD}/iot_privatekey.pem" + + echo "Success!" + return 0 +fi + +# Get the file from S3 +aws s3 cp ${testing_env_bucket} ${PWD}/environment_files.txt +testing_env_file=$( cat environment_files.txt ) +# Make sure we have data of some form +if [ "${testing_env_file}" != "" ]; then + echo "Environment variables secret found" +else + echo "Could not get environment variables from secrets!" + return 1 +fi + +# Make all the variables in mqtt5_environment_variables.txt exported +# so we can run MQTT5 tests +export $(grep -v '^#' environment_files.txt | xargs) + +# CRT/non-builder certificate and key processing +# Get the certificate and key secrets (dumps straight to a file) +crt_cert_file=$(aws secretsmanager get-secret-value --secret-id "${AWS_TEST_MQTT5_CERTIFICATE_FILE_SECRET}" --query "SecretString" --region ${region} | cut -f2 -d\") && echo "$crt_cert_file" > ${PWD}/crt_certificate.pem +crt_key_file=$(aws secretsmanager get-secret-value --secret-id "${AWS_TEST_MQTT5_KEY_FILE_SECRET}" --query "SecretString" --region ${region} | cut -f2 -d\") && echo "$crt_key_file" > ${PWD}/crt_privatekey.pem +# Does the certificate file have data? If not, then abort! +if [ "${crt_cert_file}" != "" ]; then + echo "CRT Certificate secret found" +else + echo "Could not get CRT certificate from secrets!" + + # Clean up... + unset $(grep -v '^#' environment_files.txt | xargs | cut -d "=" -f 1) + rm "${PWD}/environment_files.txt" + rm "${PWD}/crt_certificate.pem" + rm "${PWD}/crt_privatekey.pem" + + return 1 +fi +# Does the private key file have data? If not, then abort! +if [ "${crt_key_file}" != "" ]; then + echo "CRT Private key secret found" +else + echo "Could not get CRT private key from secrets!" + + # Clean up... + unset $(grep -v '^#' environment_files.txt | xargs | cut -d "=" -f 1) + rm "${PWD}/environment_files.txt" + rm "${PWD}/crt_certificate.pem" + rm "${PWD}/crt_privatekey.pem" + + return 1 +fi +# Set the certificate and key paths (absolute paths for best compatbility) +export AWS_TEST_MQTT5_CERTIFICATE_FILE="${PWD}/crt_certificate.pem" +export AWS_TEST_MQTT5_KEY_FILE="${PWD}/crt_privatekey.pem" + + +# IoT/Builder certificate and key processing +# Get the certificate and key secrets (dumps straight to a file) +iot_cert_file=$(aws secretsmanager get-secret-value --secret-id "${AWS_TEST_MQTT5_IOT_CERTIFICATE_PATH_SECRET}" --region ${region} --query "SecretString" | cut -f2 -d":" | cut -f2 -d\") && echo -e "$iot_cert_file" > ${PWD}/iot_certificate.pem +iot_key_file=$(aws secretsmanager get-secret-value --secret-id "${AWS_TEST_MQTT5_IOT_KEY_PATH_SECRET}" --region ${region} --query "SecretString" | cut -f2 -d":" | cut -f2 -d\") && echo -e "$iot_key_file" > ${PWD}/iot_privatekey.pem +# Does the certificate file have data? If not, then abort! +if [ "${iot_cert_file}" != "" ]; then + echo "IoT Certificate secret found" +else + echo "Could not get IoT certificate from secrets!" + + # Clean up... + unset $(grep -v '^#' environment_files.txt | xargs | cut -d "=" -f 1) + unset AWS_TEST_MQTT5_CERTIFICATE_FILE + unset AWS_TEST_MQTT5_KEY_FILE + rm "${PWD}/environment_files.txt" + rm "${PWD}/crt_certificate.pem" + rm "${PWD}/crt_privatekey.pem" + rm "${PWD}/iot_certificate.pem" + rm "${PWD}/iot_privatekey.pem" + + return 1 +fi +# Does the private key file have data? If not, then abort! +if [ "${iot_key_file}" != "" ]; then + echo "IoT Private key secret found" +else + echo "Could not get IoT private key from secrets!" + + # Clean up... + unset $(grep -v '^#' environment_files.txt | xargs | cut -d "=" -f 1) + unset AWS_TEST_MQTT5_CERTIFICATE_FILE + unset AWS_TEST_MQTT5_KEY_FILE + rm "${PWD}/environment_files.txt" + rm "${PWD}/crt_certificate.pem" + rm "${PWD}/crt_privatekey.pem" + rm "${PWD}/iot_certificate.pem" + rm "${PWD}/iot_privatekey.pem" + + return 1 +fi +# Set IoT certificate and key paths +export AWS_TEST_MQTT5_IOT_CERTIFICATE_PATH="${PWD}/iot_certificate.pem" +export AWS_TEST_MQTT5_IOT_KEY_PATH="${PWD}/iot_privatekey.pem" + +# Everything is set +echo "Success: Environment variables set!" From f1daf3c756be8dd117188d0e33046af2af5b34ab Mon Sep 17 00:00:00 2001 From: Steve Kim Date: Thu, 1 Dec 2022 09:36:52 -0800 Subject: [PATCH 14/61] comment out properly --- .github/workflows/ci.yml | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4c713db1..c9441d52 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -97,19 +97,17 @@ jobs: run: | python -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder.pyz')" python builder.pyz build -p ${{ env.PACKAGE_NAME }} - - - name: configure AWS credentials (MQTT5) - uses: aws-actions/configure-aws-credentials@v1 - with: - role-to-assume: ${{ env.CI_MQTT5_ROLE }} - aws-region: ${{ env.AWS_DEFAULT_REGION }} + # - name: configure AWS credentials (MQTT5) + # uses: aws-actions/configure-aws-credentials@v1 + # with: + # role-to-assume: ${{ env.CI_MQTT5_ROLE }} + # aws-region: ${{ env.AWS_DEFAULT_REGION }} # - name: MQTT5 tests # shell: bash # run: | # source utils/mqtt5_test_setup.sh s3://iot-sdk-ci-bucket-us-east1/IotUsProdMqtt5EnvironmentVariables.txt us-east-1 # mvn test -Dtest=Mqtt5BuilderTest -DfailIfNoTests=false # source utils/mqtt5_test_setup.sh s3://iot-sdk-ci-bucket-us-east1/IotUsProdMqtt5EnvironmentVariables.txt cleanup - - name: Running samples in CI setup run: | python -m pip install boto3 From a344550ea7621cd06688ec18c731808395accf71 Mon Sep 17 00:00:00 2001 From: Steve Kim Date: Thu, 1 Dec 2022 09:48:59 -0800 Subject: [PATCH 15/61] fix ci.yml --- .github/workflows/ci.yml | 22 +--------------------- 1 file changed, 1 insertion(+), 21 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c9441d52..a48d9520 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -97,17 +97,6 @@ jobs: run: | python -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder.pyz')" python builder.pyz build -p ${{ env.PACKAGE_NAME }} - # - name: configure AWS credentials (MQTT5) - # uses: aws-actions/configure-aws-credentials@v1 - # with: - # role-to-assume: ${{ env.CI_MQTT5_ROLE }} - # aws-region: ${{ env.AWS_DEFAULT_REGION }} - # - name: MQTT5 tests - # shell: bash - # run: | - # source utils/mqtt5_test_setup.sh s3://iot-sdk-ci-bucket-us-east1/IotUsProdMqtt5EnvironmentVariables.txt us-east-1 - # mvn test -Dtest=Mqtt5BuilderTest -DfailIfNoTests=false - # source utils/mqtt5_test_setup.sh s3://iot-sdk-ci-bucket-us-east1/IotUsProdMqtt5EnvironmentVariables.txt cleanup - name: Running samples in CI setup run: | python -m pip install boto3 @@ -156,9 +145,6 @@ jobs: - name: run PubSub sample run: | python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --language Python --sample_file "${{ env.CI_SAMPLES_FOLDER }}/pubsub.py" --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/PubSub/cert' --sample_secret_private_key 'ci/PubSub/key' - - name: run MQTT5 PubSub sample - run: | - python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --language Python --sample_file "${{ env.CI_SAMPLES_FOLDER }}/mqtt5_pubsub.py" --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/PubSub/cert' --sample_secret_private_key 'ci/PubSub/key' - name: configure AWS credentials (Device Advisor) uses: aws-actions/configure-aws-credentials@v1 with: @@ -190,9 +176,6 @@ jobs: - name: run PubSub sample run: | python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --language Python --sample_file "${{ env.CI_SAMPLES_FOLDER }}/pubsub.py" --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/PubSub/cert' --sample_secret_private_key 'ci/PubSub/key' - - name: run MQTT5 PubSub sample - run: | - python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --language Python --sample_file "${{ env.CI_SAMPLES_FOLDER }}/mqtt5_pubsub.py" --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/PubSub/cert' --sample_secret_private_key 'ci/PubSub/key' - name: configure AWS credentials (Device Advisor) uses: aws-actions/configure-aws-credentials@v1 with: @@ -234,10 +217,7 @@ jobs: - name: run PubSub sample run: | python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --language Python --sample_file "${{ env.CI_SAMPLES_FOLDER }}/pubsub.py" --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/PubSub/cert' --sample_secret_private_key 'ci/PubSub/key' - - name: run MQTT5 PubSub sample - run: | - python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --language Python --sample_file "${{ env.CI_SAMPLES_FOLDER }}/mqtt5_pubsub.py" --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/PubSub/cert' --sample_secret_private_key 'ci/PubSub/key' - - name: run PKCS11 Connect sample + - name: run PKCS11 Connect sample run: | mkdir -p /tmp/tokens export SOFTHSM2_CONF=/tmp/softhsm2.conf From 444bf90ad4848c2ec657ae8fe17edc170ae76f04 Mon Sep 17 00:00:00 2001 From: Steve Kim Date: Thu, 1 Dec 2022 09:50:49 -0800 Subject: [PATCH 16/61] add mqtt5_pubsub sample to all jobs --- .github/workflows/ci.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a48d9520..d1f21d45 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -145,6 +145,9 @@ jobs: - name: run PubSub sample run: | python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --language Python --sample_file "${{ env.CI_SAMPLES_FOLDER }}/pubsub.py" --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/PubSub/cert' --sample_secret_private_key 'ci/PubSub/key' + - name: run MQTT5 PubSub sample + run: | + python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --language Python --sample_file "${{ env.CI_SAMPLES_FOLDER }}/mqtt5_pubsub.py" --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/PubSub/cert' --sample_secret_private_key 'ci/PubSub/key' - name: configure AWS credentials (Device Advisor) uses: aws-actions/configure-aws-credentials@v1 with: @@ -176,6 +179,9 @@ jobs: - name: run PubSub sample run: | python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --language Python --sample_file "${{ env.CI_SAMPLES_FOLDER }}/pubsub.py" --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/PubSub/cert' --sample_secret_private_key 'ci/PubSub/key' + - name: run MQTT5 PubSub sample + run: | + python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --language Python --sample_file "${{ env.CI_SAMPLES_FOLDER }}/mqtt5_pubsub.py" --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/PubSub/cert' --sample_secret_private_key 'ci/PubSub/key' - name: configure AWS credentials (Device Advisor) uses: aws-actions/configure-aws-credentials@v1 with: @@ -217,6 +223,9 @@ jobs: - name: run PubSub sample run: | python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --language Python --sample_file "${{ env.CI_SAMPLES_FOLDER }}/pubsub.py" --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/PubSub/cert' --sample_secret_private_key 'ci/PubSub/key' + - name: run MQTT5 PubSub sample + run: | + python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --language Python --sample_file "${{ env.CI_SAMPLES_FOLDER }}/mqtt5_pubsub.py" --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/PubSub/cert' --sample_secret_private_key 'ci/PubSub/key' - name: run PKCS11 Connect sample run: | mkdir -p /tmp/tokens From 9ab83d320ff2ea16caa7831e05c94ae53dea92dd Mon Sep 17 00:00:00 2001 From: Steve Kim Date: Thu, 1 Dec 2022 09:55:13 -0800 Subject: [PATCH 17/61] Add 'is_ci' related command to sample --- samples/mqtt5_pubsub.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/samples/mqtt5_pubsub.py b/samples/mqtt5_pubsub.py index b171cb7e..f2888c6c 100644 --- a/samples/mqtt5_pubsub.py +++ b/samples/mqtt5_pubsub.py @@ -35,6 +35,7 @@ "The number of messages to send (optional, default='10').", default=10, type=int) +cmdUtils.register_command("is_ci", "", "If present the sample will run in CI mode (optional, default='None')") # Needs to be called so the command utils parse the commands cmdUtils.get_args() @@ -42,6 +43,7 @@ received_all_event = threading.Event() future_stopped = Future() future_connection_success = Future() +is_ci = cmdUtils.get_command("is_ci", None) != None # Callback when any publish is received @@ -89,6 +91,12 @@ def on_lifecycle_connection_failure(lifecycle_connection_failure: mqtt5.Lifecycl on_lifecycle_connection_failure=on_lifecycle_connection_failure) print("MQTT5 Client Created") + if is_ci == False: + print("Connecting to {} with client ID '{}'...".format( + cmdUtils.get_command(cmdUtils.m_cmd_endpoint), cmdUtils.get_command("client_id"))) + else: + print("Connecting to endpoint with client ID") + client.start() lifecycle_connect_success_data = future_connection_success.result(TIMEOUT) connack_packet = lifecycle_connect_success_data.connack_packet From ea3eb95172666d78bea6166148fb51d9a7a40754 Mon Sep 17 00:00:00 2001 From: Steve Kim Date: Thu, 1 Dec 2022 10:41:41 -0800 Subject: [PATCH 18/61] mqtt5 builder basic test --- test/test_mqtt5.py | 109 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 109 insertions(+) create mode 100644 test/test_mqtt5.py diff --git a/test/test_mqtt5.py b/test/test_mqtt5.py new file mode 100644 index 00000000..07e64278 --- /dev/null +++ b/test/test_mqtt5.py @@ -0,0 +1,109 @@ +from awscrt import mqtt5 +from awscrt.auth import AwsCredentialsProvider +from awscrt.io import ClientBootstrap, DefaultHostResolver, EventLoopGroup +from awsiot import mqtt5_client_builder +from concurrent.futures import Future +import boto3 +import botocore.exceptions +import os +import unittest +import shutil +import tempfile +import uuid +import warnings + +TIMEOUT = 100.0 +PROXY_HOST = os.environ.get('proxyhost') +PROXY_PORT = int(os.environ.get('proxyport', '0')) + + +class Config: + cache = None + + def __init__(self, endpoint, cert, key, region, cognito_creds): + self.cert = cert + self.key = key + self.endpoint = endpoint + self.region = region + self.cognito_creds = cognito_creds + + @staticmethod + def get(): + """Raises SkipTest if credentials aren't set up correctly""" + if Config.cache: + return Config.cache + + # boto3 caches the HTTPS connection for the API calls, which appears to the unit test + # framework as a leak, so ignore it, that's not what we're testing here + warnings.simplefilter('ignore', ResourceWarning) + + try: + secrets = boto3.client('secretsmanager') + response = secrets.get_secret_value(SecretId='unit-test/endpoint') + endpoint = response['SecretString'] + response = secrets.get_secret_value(SecretId='unit-test/certificate') + cert = response['SecretString'].encode('utf8') + response = secrets.get_secret_value(SecretId='unit-test/privatekey') + key = response['SecretString'].encode('utf8') + region = secrets.meta.region_name + response = secrets.get_secret_value(SecretId='unit-test/cognitopool') + cognito_pool = response['SecretString'] + + cognito = boto3.client('cognito-identity') + response = cognito.get_id(IdentityPoolId=cognito_pool) + cognito_id = response['IdentityId'] + response = cognito.get_credentials_for_identity(IdentityId=cognito_id) + cognito_creds = response['Credentials'] + + Config.cache = Config(endpoint, cert, key, region, cognito_creds) + except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as ex: + raise unittest.SkipTest("No credentials") + + return Config.cache + + +def create_client_id(): + return 'aws-iot-device-sdk-python-v2-unit-test-{0}'.format(uuid.uuid4()) + +class Mqtt5TestCallbacks(): + def __init__(self): + self.future_connection_success = Future() + self.future_stopped = Future() + + def ws_handshake_transform(self, transform_args): + transform_args.set_done() + + def on_lifecycle_stopped(self, lifecycle_stopped: mqtt5.LifecycleStoppedData): + if self.future_stopped: + self.future_stopped.set_result(None) + + def on_lifecycle_connection_success(self, lifecycle_connection_success: mqtt5.LifecycleConnectSuccessData): + self.negotiated_settings = lifecycle_connection_success.negotiated_settings + self.connack_packet = lifecycle_connection_success.connack_packet + if self.future_connection_success: + self.future_connection_success.set_result(lifecycle_connection_success) + +class Mqtt5BuilderTest(unittest.TestCase): + def _test_connection(self, client: mqtt5.Client, callbacks: Mqtt5TestCallbacks): + client.start() + callbacks.future_connection_success.result(TIMEOUT) + client.stop() + callbacks.future_stopped(TIMEOUT) + + def test_mtls_from_bytes(self): + config = Config.get() + elg = EventLoopGroup() + resolver = DefaultHostResolver(elg) + bootstrap = ClientBootstrap(elg, resolver) + callbacks = Mqtt5TestCallbacks() + + client = mqtt5_client_builder.mtls_from_bytes( + cert_bytes=config.cert, + pri_key_bytes=config.key, + endpoint=config.endpoint, + client_id=create_client_id(), + client_bootstrap=bootstrap, + on_lifecycle_connection_success=callbacks.on_lifecycle_connection_success, + on_lifecycle_stopped=callbacks.on_lifecycle_stopped) + + self._test_connection(self, client, callbacks) From 203315b823f0b06151ae6f20a18019cfa9322074 Mon Sep 17 00:00:00 2001 From: Steve Kim Date: Thu, 1 Dec 2022 13:50:18 -0800 Subject: [PATCH 19/61] added mqtt5 env setup to ci --- .github/workflows/ci.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d1f21d45..f965a25c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -40,10 +40,16 @@ jobs: with: role-to-assume: ${{ env.CI_IOT_CONTAINERS }} aws-region: ${{ env.AWS_DEFAULT_REGION }} + - name: configure AWS credentials (MQTT5) + uses: aws-actions/configure-aws-credentials@v1 + with: + role-to-assume: ${{ env.CI_MQTT5_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} - name: Build ${{ env.PACKAGE_NAME }} # There's hackery in builder.json so that when we run on manylinux # we build and test using every version of python that we support. run: | + source utils/mqtt5_test_setup.sh s3://iot-sdk-ci-bucket-us-east1/IotUsProdMqtt5EnvironmentVariables.txt us-east-1 aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-manylinux2014-x64 build -p ${{ env.PACKAGE_NAME }} From ed99b82f5dcee5b7f2c1e8ef08f19e6483748e11 Mon Sep 17 00:00:00 2001 From: Steve Kim Date: Thu, 1 Dec 2022 13:52:01 -0800 Subject: [PATCH 20/61] re-order mqtt5 env setup --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f965a25c..adfd4a04 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -49,8 +49,8 @@ jobs: # There's hackery in builder.json so that when we run on manylinux # we build and test using every version of python that we support. run: | - source utils/mqtt5_test_setup.sh s3://iot-sdk-ci-bucket-us-east1/IotUsProdMqtt5EnvironmentVariables.txt us-east-1 aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh + source utils/mqtt5_test_setup.sh s3://iot-sdk-ci-bucket-us-east1/IotUsProdMqtt5EnvironmentVariables.txt us-east-1 ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-manylinux2014-x64 build -p ${{ env.PACKAGE_NAME }} al2: From a6e4b8e4cc81bd123cbb3b8f67630aac68e4b911 Mon Sep 17 00:00:00 2001 From: Steve Kim Date: Thu, 1 Dec 2022 13:53:25 -0800 Subject: [PATCH 21/61] remove execute of mqtt5 env variables setup --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index adfd4a04..14d9b2e9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -50,7 +50,7 @@ jobs: # we build and test using every version of python that we support. run: | aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh - source utils/mqtt5_test_setup.sh s3://iot-sdk-ci-bucket-us-east1/IotUsProdMqtt5EnvironmentVariables.txt us-east-1 + # source utils/mqtt5_test_setup.sh s3://iot-sdk-ci-bucket-us-east1/IotUsProdMqtt5EnvironmentVariables.txt us-east-1 ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-manylinux2014-x64 build -p ${{ env.PACKAGE_NAME }} al2: From b8bb10d45b352a125204ebc2c97bf97ade4e1641 Mon Sep 17 00:00:00 2001 From: Steve Kim Date: Thu, 1 Dec 2022 13:55:30 -0800 Subject: [PATCH 22/61] remove mqtt5 env var setup --- .github/workflows/ci.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 14d9b2e9..e6f301ea 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -40,11 +40,6 @@ jobs: with: role-to-assume: ${{ env.CI_IOT_CONTAINERS }} aws-region: ${{ env.AWS_DEFAULT_REGION }} - - name: configure AWS credentials (MQTT5) - uses: aws-actions/configure-aws-credentials@v1 - with: - role-to-assume: ${{ env.CI_MQTT5_ROLE }} - aws-region: ${{ env.AWS_DEFAULT_REGION }} - name: Build ${{ env.PACKAGE_NAME }} # There's hackery in builder.json so that when we run on manylinux # we build and test using every version of python that we support. From 44143cee7ee8353672eedcbb3145176e3b84b06f Mon Sep 17 00:00:00 2001 From: Steve Kim Date: Thu, 1 Dec 2022 14:13:03 -0800 Subject: [PATCH 23/61] access keys --- .github/workflows/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e6f301ea..de5fc9b8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -28,6 +28,8 @@ env: CI_FLEET_PROVISIONING_ROLE: ${{ secrets.AWS_CI_FLEET_PROVISIONING_ROLE }} CI_DEVICE_ADVISOR: ${{ secrets.AWS_CI_DEVICE_ADVISOR_ROLE }} CI_MQTT5_ROLE: ${{ secrets.AWS_CI_MQTT5_ROLE }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_DATEST_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_DATEST_SECRET_ACCESS_KEY }} jobs: all-python-versions: From 0f446d26c4d57a4423b0b1c70f2b6c29d986c156 Mon Sep 17 00:00:00 2001 From: Steve Kim Date: Thu, 1 Dec 2022 14:16:11 -0800 Subject: [PATCH 24/61] remove commented line from ci.yml --- .github/workflows/ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index de5fc9b8..a00c52fd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -47,7 +47,6 @@ jobs: # we build and test using every version of python that we support. run: | aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh - # source utils/mqtt5_test_setup.sh s3://iot-sdk-ci-bucket-us-east1/IotUsProdMqtt5EnvironmentVariables.txt us-east-1 ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-manylinux2014-x64 build -p ${{ env.PACKAGE_NAME }} al2: From 1ccd19e1dac91dace91af924e2d80de65a4c2c2f Mon Sep 17 00:00:00 2001 From: Steve Kim Date: Thu, 1 Dec 2022 14:20:17 -0800 Subject: [PATCH 25/61] removed CI_MQTT5_ROLE --- .github/workflows/ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a00c52fd..f33407c7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,7 +27,6 @@ env: CI_JOBS_ROLE: ${{ secrets.AWS_CI_JOBS_ROLE }} CI_FLEET_PROVISIONING_ROLE: ${{ secrets.AWS_CI_FLEET_PROVISIONING_ROLE }} CI_DEVICE_ADVISOR: ${{ secrets.AWS_CI_DEVICE_ADVISOR_ROLE }} - CI_MQTT5_ROLE: ${{ secrets.AWS_CI_MQTT5_ROLE }} AWS_ACCESS_KEY_ID: ${{ secrets.AWS_DATEST_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_DATEST_SECRET_ACCESS_KEY }} From 1f8681c483af6efae3d948c5a4f2b8a7640a9cd3 Mon Sep 17 00:00:00 2001 From: Steve Kim Date: Thu, 1 Dec 2022 14:29:06 -0800 Subject: [PATCH 26/61] updated env variable to show ROLE --- .github/workflows/ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f33407c7..b2760ffe 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,7 +20,7 @@ env: DA_SHADOW_VALUE_DEFAULT: OFF CI_UTILS_FOLDER: "./aws-iot-device-sdk-python-v2/utils" CI_SAMPLES_FOLDER: "./aws-iot-device-sdk-python-v2/samples" - CI_IOT_CONTAINERS: ${{ secrets.AWS_CI_IOT_CONTAINERS }} + CI_IOT_CONTAINERS_ROLE: ${{ secrets.AWS_CI_IOT_CONTAINERS }} CI_PUBSUB_ROLE: ${{ secrets.AWS_CI_PUBSUB_ROLE }} CI_CUSTOM_AUTHORIZER_ROLE: ${{ secrets.AWS_CI_CUSTOM_AUTHORIZER_ROLE }} CI_SHADOW_ROLE: ${{ secrets.AWS_CI_SHADOW_ROLE }} @@ -39,7 +39,7 @@ jobs: - name: configure AWS credentials (containers) uses: aws-actions/configure-aws-credentials@v1 with: - role-to-assume: ${{ env.CI_IOT_CONTAINERS }} + role-to-assume: ${{ env.CI_IOT_CONTAINERS_ROLE }} aws-region: ${{ env.AWS_DEFAULT_REGION }} - name: Build ${{ env.PACKAGE_NAME }} # There's hackery in builder.json so that when we run on manylinux @@ -56,7 +56,7 @@ jobs: - name: configure AWS credentials (containers) uses: aws-actions/configure-aws-credentials@v1 with: - role-to-assume: ${{ env.CI_IOT_CONTAINERS }} + role-to-assume: ${{ env.CI_IOT_CONTAINERS_ROLE }} aws-region: ${{ env.AWS_DEFAULT_REGION }} # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages - name: Build ${{ env.PACKAGE_NAME }} @@ -78,7 +78,7 @@ jobs: - name: configure AWS credentials (containers) uses: aws-actions/configure-aws-credentials@v1 with: - role-to-assume: ${{ env.CI_IOT_CONTAINERS }} + role-to-assume: ${{ env.CI_IOT_CONTAINERS_ROLE }} aws-region: ${{ env.AWS_DEFAULT_REGION }} # set arm arch - name: Install qemu/docker From 890654f47e813a21fc137a6dacc6e443a4771702 Mon Sep 17 00:00:00 2001 From: Steve Kim Date: Thu, 1 Dec 2022 14:35:04 -0800 Subject: [PATCH 27/61] changed variable name --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b2760ffe..b2769055 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -28,7 +28,7 @@ env: CI_FLEET_PROVISIONING_ROLE: ${{ secrets.AWS_CI_FLEET_PROVISIONING_ROLE }} CI_DEVICE_ADVISOR: ${{ secrets.AWS_CI_DEVICE_ADVISOR_ROLE }} AWS_ACCESS_KEY_ID: ${{ secrets.AWS_DATEST_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_DATEST_SECRET_ACCESS_KEY }} + AWS_SECRET_ACCESS: ${{ secrets.AWS_DATEST_SECRET_ACCESS_KEY }} jobs: all-python-versions: From 1bcc38b321bfb2262c7a2a022aad224295bc7234 Mon Sep 17 00:00:00 2001 From: Steve Kim Date: Thu, 1 Dec 2022 14:42:54 -0800 Subject: [PATCH 28/61] remove mqtt5 sample from linux job --- .github/workflows/ci.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b2769055..e8b73914 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -180,9 +180,6 @@ jobs: - name: run PubSub sample run: | python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --language Python --sample_file "${{ env.CI_SAMPLES_FOLDER }}/pubsub.py" --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/PubSub/cert' --sample_secret_private_key 'ci/PubSub/key' - - name: run MQTT5 PubSub sample - run: | - python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --language Python --sample_file "${{ env.CI_SAMPLES_FOLDER }}/mqtt5_pubsub.py" --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/PubSub/cert' --sample_secret_private_key 'ci/PubSub/key' - name: configure AWS credentials (Device Advisor) uses: aws-actions/configure-aws-credentials@v1 with: From 2bd54d80300ce64c19af99a772e74c2c188a8fca Mon Sep 17 00:00:00 2001 From: Steve Kim Date: Thu, 1 Dec 2022 14:44:18 -0800 Subject: [PATCH 29/61] re-added mqtt5 sample test to linux --- .github/workflows/ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e8b73914..b2769055 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -180,6 +180,9 @@ jobs: - name: run PubSub sample run: | python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --language Python --sample_file "${{ env.CI_SAMPLES_FOLDER }}/pubsub.py" --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/PubSub/cert' --sample_secret_private_key 'ci/PubSub/key' + - name: run MQTT5 PubSub sample + run: | + python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --language Python --sample_file "${{ env.CI_SAMPLES_FOLDER }}/mqtt5_pubsub.py" --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/PubSub/cert' --sample_secret_private_key 'ci/PubSub/key' - name: configure AWS credentials (Device Advisor) uses: aws-actions/configure-aws-credentials@v1 with: From 739838ba1fb7f4af3884f9206fee99bc1850dafa Mon Sep 17 00:00:00 2001 From: Steve Kim Date: Thu, 1 Dec 2022 14:46:30 -0800 Subject: [PATCH 30/61] changed from boto3 to boto in linux ci --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b2769055..46f2d303 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -171,7 +171,7 @@ jobs: ./builder build -p ${{ env.PACKAGE_NAME }} - name: Running samples in CI setup run: | - python3 -m pip install boto3 + python3 -m pip install boto - name: configure AWS credentials (PubSub) uses: aws-actions/configure-aws-credentials@v1 with: From 61586e928340a81cc504632cf78febd27c834e6f Mon Sep 17 00:00:00 2001 From: Steve Kim Date: Thu, 1 Dec 2022 14:48:29 -0800 Subject: [PATCH 31/61] reverted to boto3 --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 46f2d303..b2769055 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -171,7 +171,7 @@ jobs: ./builder build -p ${{ env.PACKAGE_NAME }} - name: Running samples in CI setup run: | - python3 -m pip install boto + python3 -m pip install boto3 - name: configure AWS credentials (PubSub) uses: aws-actions/configure-aws-credentials@v1 with: From 535627244255b86ed7e7bbc38e43016f5408f109 Mon Sep 17 00:00:00 2001 From: Steve Kim Date: Thu, 1 Dec 2022 15:03:10 -0800 Subject: [PATCH 32/61] fix secrets access --- .github/workflows/ci.yml | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b2769055..9defd241 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,8 +27,6 @@ env: CI_JOBS_ROLE: ${{ secrets.AWS_CI_JOBS_ROLE }} CI_FLEET_PROVISIONING_ROLE: ${{ secrets.AWS_CI_FLEET_PROVISIONING_ROLE }} CI_DEVICE_ADVISOR: ${{ secrets.AWS_CI_DEVICE_ADVISOR_ROLE }} - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_DATEST_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS: ${{ secrets.AWS_DATEST_SECRET_ACCESS_KEY }} jobs: all-python-versions: @@ -36,12 +34,10 @@ jobs: permissions: id-token: write # This is required for requesting the JWT steps: - - name: configure AWS credentials (containers) - uses: aws-actions/configure-aws-credentials@v1 - with: - role-to-assume: ${{ env.CI_IOT_CONTAINERS_ROLE }} - aws-region: ${{ env.AWS_DEFAULT_REGION }} - name: Build ${{ env.PACKAGE_NAME }} + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_DA_ROLE_KEY }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_DA_ROLE_PRIVATE_KEY }} # There's hackery in builder.json so that when we run on manylinux # we build and test using every version of python that we support. run: | From 0f787824e3a9e871d0df2b30c681e3f5df7cb306 Mon Sep 17 00:00:00 2001 From: Steve Kim Date: Thu, 1 Dec 2022 15:14:19 -0800 Subject: [PATCH 33/61] attempt to use mqtt5 script --- .github/workflows/ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9defd241..a3e121e6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,6 +34,8 @@ jobs: permissions: id-token: write # This is required for requesting the JWT steps: + - name: Checkout Sources + uses: actions/checkout@v2 - name: Build ${{ env.PACKAGE_NAME }} env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_DA_ROLE_KEY }} @@ -41,6 +43,8 @@ jobs: # There's hackery in builder.json so that when we run on manylinux # we build and test using every version of python that we support. run: | + chmod a+xr ./utils/mqtt5_test_setup.sh + source ./utils/mqtt5_test_setup.sh s3://aws-crt-test-stuff/TestIotProdMQTT5EnvironmentVariables.txt us-east-1 aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-manylinux2014-x64 build -p ${{ env.PACKAGE_NAME }} From e3c76eb5e6d232f2dbd8d1685c0fdfb36e6b85f4 Mon Sep 17 00:00:00 2001 From: Steve Kim Date: Thu, 1 Dec 2022 15:17:46 -0800 Subject: [PATCH 34/61] test print to check env variable --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a3e121e6..fee4b6e9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -47,6 +47,7 @@ jobs: source ./utils/mqtt5_test_setup.sh s3://aws-crt-test-stuff/TestIotProdMQTT5EnvironmentVariables.txt us-east-1 aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-manylinux2014-x64 build -p ${{ env.PACKAGE_NAME }} + echo ${{ AWS_TEST_MQTT5_KEY_FILE }} al2: runs-on: ubuntu-latest From 9d49a6c2cf037fd989f9fcc5ddf010017feb3851 Mon Sep 17 00:00:00 2001 From: Steve Kim Date: Thu, 1 Dec 2022 15:21:30 -0800 Subject: [PATCH 35/61] test print environ --- test/test_mqtt5.py | 53 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/test/test_mqtt5.py b/test/test_mqtt5.py index 07e64278..fc5eb0b4 100644 --- a/test/test_mqtt5.py +++ b/test/test_mqtt5.py @@ -85,6 +85,7 @@ def on_lifecycle_connection_success(self, lifecycle_connection_success: mqtt5.Li class Mqtt5BuilderTest(unittest.TestCase): def _test_connection(self, client: mqtt5.Client, callbacks: Mqtt5TestCallbacks): + print(os.environ['AWS_TEST_MQTT5_KEY_FILE']) client.start() callbacks.future_connection_success.result(TIMEOUT) client.stop() @@ -107,3 +108,55 @@ def test_mtls_from_bytes(self): on_lifecycle_stopped=callbacks.on_lifecycle_stopped) self._test_connection(self, client, callbacks) + + def test_mtls_from_path(self): + config = Config.get() + elg = EventLoopGroup() + resolver = DefaultHostResolver(elg) + bootstrap = ClientBootstrap(elg, resolver) + callbacks = Mqtt5TestCallbacks() + + # test "from path" builder by writing secrets to tempfiles + tmp_dirpath = tempfile.mkdtemp() + try: + cert_filepath = os.path.join(tmp_dirpath, 'cert') + with open(cert_filepath, 'wb') as cert_file: + cert_file.write(config.cert) + + key_filepath = os.path.join(tmp_dirpath, 'key') + with open(key_filepath, 'wb') as key_file: + key_file.write(config.key) + + client = mqtt5_client_builder.mtls_from_bytes( + cert_filepath=cert_filepath, + pri_key_filepath=key_filepath, + endpoint=config.endpoint, + client_id=create_client_id(), + client_bootstrap=bootstrap, + on_lifecycle_connection_success=callbacks.on_lifecycle_connection_success, + on_lifecycle_stopped=callbacks.on_lifecycle_stopped) + + finally: + shutil.rmtree(tmp_dirpath) + + self._test_connection(self, client, callbacks) + + def test_websockets_default(self): + """Websocket connection with default credentials provider""" + config = Config.get() + elg = EventLoopGroup() + resolver = DefaultHostResolver(elg) + bootstrap = ClientBootstrap(elg, resolver) + cred_provider = AwsCredentialsProvider.new_default_chain(bootstrap) + callbacks = Mqtt5TestCallbacks() + + client = mqtt5_client_builder.websockets_with_default_aws_signing( + region=config.region, + credentials_provider=cred_provider, + endpoint=config.endpoint, + client_id=create_client_id(), + client_bootstrap=bootstrap, + on_lifecycle_connection_success=callbacks.on_lifecycle_connection_success, + on_lifecycle_stopped=callbacks.on_lifecycle_stopped) + + self._test_connection(self, client, callbacks) \ No newline at end of file From 0df2fc9ef9ed02d3b0952a12430c83fc16b1e71b Mon Sep 17 00:00:00 2001 From: Steve Kim Date: Thu, 1 Dec 2022 15:21:54 -0800 Subject: [PATCH 36/61] fix ci.yml --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fee4b6e9..ff85e7ee 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -47,7 +47,7 @@ jobs: source ./utils/mqtt5_test_setup.sh s3://aws-crt-test-stuff/TestIotProdMQTT5EnvironmentVariables.txt us-east-1 aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-manylinux2014-x64 build -p ${{ env.PACKAGE_NAME }} - echo ${{ AWS_TEST_MQTT5_KEY_FILE }} + al2: runs-on: ubuntu-latest From 3bbbfb5450e7ae3690c990c0dd1bc7e2efeb0218 Mon Sep 17 00:00:00 2001 From: Steve Kim Date: Thu, 1 Dec 2022 15:27:33 -0800 Subject: [PATCH 37/61] env test --- test/test_mqtt5.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test/test_mqtt5.py b/test/test_mqtt5.py index fc5eb0b4..eb070b5e 100644 --- a/test/test_mqtt5.py +++ b/test/test_mqtt5.py @@ -85,12 +85,16 @@ def on_lifecycle_connection_success(self, lifecycle_connection_success: mqtt5.Li class Mqtt5BuilderTest(unittest.TestCase): def _test_connection(self, client: mqtt5.Client, callbacks: Mqtt5TestCallbacks): - print(os.environ['AWS_TEST_MQTT5_KEY_FILE']) client.start() callbacks.future_connection_success.result(TIMEOUT) client.stop() callbacks.future_stopped(TIMEOUT) + def test_environ_variable(self): + env_text = os.environ['AWS_TEST_MQTT5_KEY_FILE'] + self.assertIsNotNone(env_text) + self.assertGreater(len(env_text), 0) + def test_mtls_from_bytes(self): config = Config.get() elg = EventLoopGroup() From f12ef5bb0e2da02bf43352e60c67e223c64ba024 Mon Sep 17 00:00:00 2001 From: Steve Kim Date: Thu, 1 Dec 2022 15:33:42 -0800 Subject: [PATCH 38/61] added -e to mqtt5 test setup --- utils/mqtt5_test_setup.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/utils/mqtt5_test_setup.sh b/utils/mqtt5_test_setup.sh index 390b7fe5..fe793385 100755 --- a/utils/mqtt5_test_setup.sh +++ b/utils/mqtt5_test_setup.sh @@ -61,8 +61,8 @@ export $(grep -v '^#' environment_files.txt | xargs) # CRT/non-builder certificate and key processing # Get the certificate and key secrets (dumps straight to a file) -crt_cert_file=$(aws secretsmanager get-secret-value --secret-id "${AWS_TEST_MQTT5_CERTIFICATE_FILE_SECRET}" --query "SecretString" --region ${region} | cut -f2 -d\") && echo "$crt_cert_file" > ${PWD}/crt_certificate.pem -crt_key_file=$(aws secretsmanager get-secret-value --secret-id "${AWS_TEST_MQTT5_KEY_FILE_SECRET}" --query "SecretString" --region ${region} | cut -f2 -d\") && echo "$crt_key_file" > ${PWD}/crt_privatekey.pem +crt_cert_file=$(aws secretsmanager get-secret-value --secret-id "${AWS_TEST_MQTT5_CERTIFICATE_FILE_SECRET}" --query "SecretString" --region ${region} | cut -f2 -d\") && echo -e "$crt_cert_file" > ${PWD}/crt_certificate.pem +crt_key_file=$(aws secretsmanager get-secret-value --secret-id "${AWS_TEST_MQTT5_KEY_FILE_SECRET}" --query "SecretString" --region ${region} | cut -f2 -d\") && echo -e "$crt_key_file" > ${PWD}/crt_privatekey.pem # Does the certificate file have data? If not, then abort! if [ "${crt_cert_file}" != "" ]; then echo "CRT Certificate secret found" @@ -141,4 +141,4 @@ export AWS_TEST_MQTT5_IOT_CERTIFICATE_PATH="${PWD}/iot_certificate.pem" export AWS_TEST_MQTT5_IOT_KEY_PATH="${PWD}/iot_privatekey.pem" # Everything is set -echo "Success: Environment variables set!" +echo "Success: Environment variables set!" \ No newline at end of file From a3c2a204c42b2be6d77c81d9c10bd46971c9ddcb Mon Sep 17 00:00:00 2001 From: Steve Kim Date: Thu, 1 Dec 2022 15:37:38 -0800 Subject: [PATCH 39/61] removed mqtt5 script --- .github/workflows/ci.yml | 4 ---- test/test_mqtt5.py | 7 +------ 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ff85e7ee..19e6d983 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,8 +34,6 @@ jobs: permissions: id-token: write # This is required for requesting the JWT steps: - - name: Checkout Sources - uses: actions/checkout@v2 - name: Build ${{ env.PACKAGE_NAME }} env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_DA_ROLE_KEY }} @@ -43,8 +41,6 @@ jobs: # There's hackery in builder.json so that when we run on manylinux # we build and test using every version of python that we support. run: | - chmod a+xr ./utils/mqtt5_test_setup.sh - source ./utils/mqtt5_test_setup.sh s3://aws-crt-test-stuff/TestIotProdMQTT5EnvironmentVariables.txt us-east-1 aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-manylinux2014-x64 build -p ${{ env.PACKAGE_NAME }} diff --git a/test/test_mqtt5.py b/test/test_mqtt5.py index eb070b5e..514dcbf3 100644 --- a/test/test_mqtt5.py +++ b/test/test_mqtt5.py @@ -90,11 +90,6 @@ def _test_connection(self, client: mqtt5.Client, callbacks: Mqtt5TestCallbacks): client.stop() callbacks.future_stopped(TIMEOUT) - def test_environ_variable(self): - env_text = os.environ['AWS_TEST_MQTT5_KEY_FILE'] - self.assertIsNotNone(env_text) - self.assertGreater(len(env_text), 0) - def test_mtls_from_bytes(self): config = Config.get() elg = EventLoopGroup() @@ -131,7 +126,7 @@ def test_mtls_from_path(self): with open(key_filepath, 'wb') as key_file: key_file.write(config.key) - client = mqtt5_client_builder.mtls_from_bytes( + client = mqtt5_client_builder.mtls_from_path( cert_filepath=cert_filepath, pri_key_filepath=key_filepath, endpoint=config.endpoint, From a0bd7d75584a91d936015f4b4fff24895ddfcc4c Mon Sep 17 00:00:00 2001 From: Steve Kim Date: Thu, 1 Dec 2022 15:39:54 -0800 Subject: [PATCH 40/61] config test --- test/test_mqtt5.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/test_mqtt5.py b/test/test_mqtt5.py index 514dcbf3..35933177 100644 --- a/test/test_mqtt5.py +++ b/test/test_mqtt5.py @@ -90,6 +90,12 @@ def _test_connection(self, client: mqtt5.Client, callbacks: Mqtt5TestCallbacks): client.stop() callbacks.future_stopped(TIMEOUT) + def test_cache(self): + config = Config.get() + self.assertIsNotNone(config.cert) + self.assertIsNotNone(config.key) + self.assertIsNotNone(config.endpoint) + def test_mtls_from_bytes(self): config = Config.get() elg = EventLoopGroup() From b901449414302c59d5fe9d006b903cb7122af5ba Mon Sep 17 00:00:00 2001 From: Steve Kim Date: Thu, 1 Dec 2022 15:41:23 -0800 Subject: [PATCH 41/61] correct arguments --- test/test_mqtt5.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/test_mqtt5.py b/test/test_mqtt5.py index 35933177..ab3934fd 100644 --- a/test/test_mqtt5.py +++ b/test/test_mqtt5.py @@ -112,7 +112,7 @@ def test_mtls_from_bytes(self): on_lifecycle_connection_success=callbacks.on_lifecycle_connection_success, on_lifecycle_stopped=callbacks.on_lifecycle_stopped) - self._test_connection(self, client, callbacks) + self._test_connection(client, callbacks) def test_mtls_from_path(self): config = Config.get() @@ -144,7 +144,7 @@ def test_mtls_from_path(self): finally: shutil.rmtree(tmp_dirpath) - self._test_connection(self, client, callbacks) + self._test_connection(client, callbacks) def test_websockets_default(self): """Websocket connection with default credentials provider""" @@ -164,4 +164,4 @@ def test_websockets_default(self): on_lifecycle_connection_success=callbacks.on_lifecycle_connection_success, on_lifecycle_stopped=callbacks.on_lifecycle_stopped) - self._test_connection(self, client, callbacks) \ No newline at end of file + self._test_connection(client, callbacks) \ No newline at end of file From 91b2336e873835758ddd9f7c09d252916a0a58a8 Mon Sep 17 00:00:00 2001 From: Steve Kim Date: Thu, 1 Dec 2022 16:09:34 -0800 Subject: [PATCH 42/61] updated test --- test/test_mqtt5.py | 48 ++++++++++++++++++---------------------------- 1 file changed, 19 insertions(+), 29 deletions(-) diff --git a/test/test_mqtt5.py b/test/test_mqtt5.py index ab3934fd..a4c809c0 100644 --- a/test/test_mqtt5.py +++ b/test/test_mqtt5.py @@ -74,14 +74,10 @@ def ws_handshake_transform(self, transform_args): transform_args.set_done() def on_lifecycle_stopped(self, lifecycle_stopped: mqtt5.LifecycleStoppedData): - if self.future_stopped: - self.future_stopped.set_result(None) + self.future_stopped.set_result(None) def on_lifecycle_connection_success(self, lifecycle_connection_success: mqtt5.LifecycleConnectSuccessData): - self.negotiated_settings = lifecycle_connection_success.negotiated_settings - self.connack_packet = lifecycle_connection_success.connack_packet - if self.future_connection_success: - self.future_connection_success.set_result(lifecycle_connection_success) + self.future_connection_success.set_result(lifecycle_connection_success) class Mqtt5BuilderTest(unittest.TestCase): def _test_connection(self, client: mqtt5.Client, callbacks: Mqtt5TestCallbacks): @@ -90,29 +86,23 @@ def _test_connection(self, client: mqtt5.Client, callbacks: Mqtt5TestCallbacks): client.stop() callbacks.future_stopped(TIMEOUT) - def test_cache(self): - config = Config.get() - self.assertIsNotNone(config.cert) - self.assertIsNotNone(config.key) - self.assertIsNotNone(config.endpoint) - - def test_mtls_from_bytes(self): - config = Config.get() - elg = EventLoopGroup() - resolver = DefaultHostResolver(elg) - bootstrap = ClientBootstrap(elg, resolver) - callbacks = Mqtt5TestCallbacks() - - client = mqtt5_client_builder.mtls_from_bytes( - cert_bytes=config.cert, - pri_key_bytes=config.key, - endpoint=config.endpoint, - client_id=create_client_id(), - client_bootstrap=bootstrap, - on_lifecycle_connection_success=callbacks.on_lifecycle_connection_success, - on_lifecycle_stopped=callbacks.on_lifecycle_stopped) - - self._test_connection(client, callbacks) + # def test_mtls_from_bytes(self): + # config = Config.get() + # elg = EventLoopGroup() + # resolver = DefaultHostResolver(elg) + # bootstrap = ClientBootstrap(elg, resolver) + # callbacks = Mqtt5TestCallbacks() + + # client = mqtt5_client_builder.mtls_from_bytes( + # cert_bytes=config.cert, + # pri_key_bytes=config.key, + # endpoint=config.endpoint, + # client_id=create_client_id(), + # client_bootstrap=bootstrap, + # on_lifecycle_connection_success=callbacks.on_lifecycle_connection_success, + # on_lifecycle_stopped=callbacks.on_lifecycle_stopped) + + # self._test_connection(client, callbacks) def test_mtls_from_path(self): config = Config.get() From 5091656d8383251492360d7c58f004da162fe9d2 Mon Sep 17 00:00:00 2001 From: Steve Kim Date: Thu, 1 Dec 2022 16:12:08 -0800 Subject: [PATCH 43/61] fixed future check --- test/test_mqtt5.py | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/test/test_mqtt5.py b/test/test_mqtt5.py index a4c809c0..22faffcc 100644 --- a/test/test_mqtt5.py +++ b/test/test_mqtt5.py @@ -84,25 +84,25 @@ def _test_connection(self, client: mqtt5.Client, callbacks: Mqtt5TestCallbacks): client.start() callbacks.future_connection_success.result(TIMEOUT) client.stop() - callbacks.future_stopped(TIMEOUT) - - # def test_mtls_from_bytes(self): - # config = Config.get() - # elg = EventLoopGroup() - # resolver = DefaultHostResolver(elg) - # bootstrap = ClientBootstrap(elg, resolver) - # callbacks = Mqtt5TestCallbacks() - - # client = mqtt5_client_builder.mtls_from_bytes( - # cert_bytes=config.cert, - # pri_key_bytes=config.key, - # endpoint=config.endpoint, - # client_id=create_client_id(), - # client_bootstrap=bootstrap, - # on_lifecycle_connection_success=callbacks.on_lifecycle_connection_success, - # on_lifecycle_stopped=callbacks.on_lifecycle_stopped) - - # self._test_connection(client, callbacks) + callbacks.future_stopped.result(TIMEOUT) + + def test_mtls_from_bytes(self): + config = Config.get() + elg = EventLoopGroup() + resolver = DefaultHostResolver(elg) + bootstrap = ClientBootstrap(elg, resolver) + callbacks = Mqtt5TestCallbacks() + + client = mqtt5_client_builder.mtls_from_bytes( + cert_bytes=config.cert, + pri_key_bytes=config.key, + endpoint=config.endpoint, + client_id=create_client_id(), + client_bootstrap=bootstrap, + on_lifecycle_connection_success=callbacks.on_lifecycle_connection_success, + on_lifecycle_stopped=callbacks.on_lifecycle_stopped) + + self._test_connection(client, callbacks) def test_mtls_from_path(self): config = Config.get() From 2d4b4a4f6c52326514a02cd71b50d0172d3aa793 Mon Sep 17 00:00:00 2001 From: Steve Kim Date: Thu, 1 Dec 2022 16:19:02 -0800 Subject: [PATCH 44/61] added remaining builder tests --- test/test_mqtt5.py | 46 +++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/test/test_mqtt5.py b/test/test_mqtt5.py index 22faffcc..abc9e15f 100644 --- a/test/test_mqtt5.py +++ b/test/test_mqtt5.py @@ -1,4 +1,4 @@ -from awscrt import mqtt5 +from awscrt import mqtt5, http from awscrt.auth import AwsCredentialsProvider from awscrt.io import ClientBootstrap, DefaultHostResolver, EventLoopGroup from awsiot import mqtt5_client_builder @@ -154,4 +154,48 @@ def test_websockets_default(self): on_lifecycle_connection_success=callbacks.on_lifecycle_connection_success, on_lifecycle_stopped=callbacks.on_lifecycle_stopped) + self._test_connection(client, callbacks) + + def test_websockets_sts(self): + """Websocket connection with X-Amz-Security-Token query param""" + config = Config.get() + elg = EventLoopGroup() + resolver = DefaultHostResolver(elg) + bootstrap = ClientBootstrap(elg, resolver) + cred_provider = AwsCredentialsProvider.new_static( + access_key_id=config.cognito_creds['AccessKeyId'], + secret_access_key=config.cognito_creds['SecretKey'], + session_token=config.cognito_creds['SessionToken']) + callbacks = Mqtt5TestCallbacks() + + client = mqtt5_client_builder.websockets_with_default_aws_signing( + region=config.region, + credentials_provider=cred_provider, + endpoint=config.endpoint, + client_id=create_client_id(), + client_bootstrap=bootstrap, + on_lifecycle_connection_success=callbacks.on_lifecycle_connection_success, + on_lifecycle_stopped=callbacks.on_lifecycle_stopped) + + self._test_connection(client, callbacks) + + @unittest.skipIf(PROXY_HOST is None, 'requires "proxyhost" and "proxyport" env vars') + def test_websockets_proxy(self): + config = Config.get() + elg = EventLoopGroup() + resolver = DefaultHostResolver(elg) + bootstrap = ClientBootstrap(elg, resolver) + cred_provider = AwsCredentialsProvider.new_default_chain(bootstrap) + callbacks = Mqtt5TestCallbacks() + + client = mqtt5_client_builder.websockets_with_default_aws_signing( + region=config.region, + credentials_provider=cred_provider, + endpoint=config.endpoint, + websocket_proxy_options=http.HttpProxyOptions(PROXY_HOST, PROXY_PORT), + client_id=create_client_id(), + client_bootstrap=bootstrap, + on_lifecycle_connection_success=callbacks.on_lifecycle_connection_success, + on_lifecycle_stopped=callbacks.on_lifecycle_stopped) + self._test_connection(client, callbacks) \ No newline at end of file From 1ec6f6f8fa049d2d8124b679736f039ba102dd0f Mon Sep 17 00:00:00 2001 From: Steve Kim Date: Thu, 1 Dec 2022 16:37:17 -0800 Subject: [PATCH 45/61] mqtt5 pkcs11 sample and test --- .github/workflows/ci.yml | 6 +++ samples/mqtt5_pkcs11_connect.py | 82 +++++++++++++++++++++++++++++++++ 2 files changed, 88 insertions(+) create mode 100644 samples/mqtt5_pkcs11_connect.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 19e6d983..2bd0419b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -230,6 +230,12 @@ jobs: export SOFTHSM2_CONF=/tmp/softhsm2.conf echo "directories.tokendir = /tmp/tokens" > /tmp/softhsm2.conf python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --language Python --sample_file "${{ env.CI_SAMPLES_FOLDER }}/pkcs11_connect.py" --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/PubSub/cert' --sample_secret_private_key 'ci/PubSub/keyp8' --sample_run_softhsm 'true' --sample_arguments '--pkcs11_lib /usr/lib/softhsm/libsofthsm2.so --pin 0000 --token_label my-token --key_label my-key' + - name: run MQTT5 PKCS11 Connect sample + run: | + mkdir -p /tmp/tokens + export SOFTHSM2_CONF=/tmp/softhsm2.conf + echo "directories.tokendir = /tmp/tokens" > /tmp/softhsm2.conf + python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --language Python --sample_file "${{ env.CI_SAMPLES_FOLDER }}/mqtt5_pkcs11_connect.py" --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/PubSub/cert' --sample_secret_private_key 'ci/PubSub/keyp8' --sample_run_softhsm 'true' --sample_arguments '--pkcs11_lib /usr/lib/softhsm/libsofthsm2.so --pin 0000 --token_label my-token --key_label my-key' - name: configure AWS credentials (Custom Authorizer) uses: aws-actions/configure-aws-credentials@v1 with: diff --git a/samples/mqtt5_pkcs11_connect.py b/samples/mqtt5_pkcs11_connect.py new file mode 100644 index 00000000..15488460 --- /dev/null +++ b/samples/mqtt5_pkcs11_connect.py @@ -0,0 +1,82 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0. + +import command_line_utils +from awscrt import mqtt5, exceptions +from uuid import uuid4 +import threading +from concurrent.futures import Future +import time + +TIMEOUT = 100 +topic_filter = "test/topic" + +# Parse arguments +cmdUtils = command_line_utils.CommandLineUtils("PubSub - Send and receive messages through an MQTT5 connection.") +cmdUtils.add_common_mqtt5_commands() +cmdUtils.add_common_proxy_commands() +cmdUtils.add_common_logging_commands() +cmdUtils.register_command("key", "", "Path to your key in PEM format.", True, str) +cmdUtils.register_command("cert", "", "Path to your client certificate in PEM format.", True, str) +cmdUtils.register_command( + "port", + "", + "Connection port. AWS IoT supports 433 and 8883 (optional, default=auto).", + type=int) +cmdUtils.register_command( + "client_id", + "", + "Client ID to use for MQTT5 connection (optional, default=None).", + default="test-" + str(uuid4())) +cmdUtils.register_command("pkcs11_lib", "", "Path to PKCS#11 Library", required=True) +cmdUtils.register_command("pin", "", "User PIN for logging into PKCS#11 token.", required=True) +cmdUtils.register_command("token_label", "", "Label of the PKCS#11 token to use (optional).") +cmdUtils.register_command("slot_id", "", "Slot ID containing the PKCS#11 token to use (optional).", False, int) +cmdUtils.register_command("key_label", "", "Label of private key on the PKCS#11 token (optional).") +cmdUtils.register_command("is_ci", "", "If present the sample will run in CI mode (optional, default='None')") +# Needs to be called so the command utils parse the commands +cmdUtils.get_args() + +future_stopped = Future() +future_connection_success = Future() +is_ci = cmdUtils.get_command("is_ci", None) != None + +# Callback when any publish is received + +# Callback for the lifecycle event Stopped +def on_lifecycle_stopped(lifecycle_stopped_data: mqtt5.LifecycleStoppedData): + print("Lifecycle Stopped") + global future_stopped + future_stopped.set_result(lifecycle_stopped_data) + + +# Callback for the lifecycle event Connection Success +def on_lifecycle_connection_success(lifecycle_connect_success_data: mqtt5.LifecycleConnectSuccessData): + print("Lifecycle Connection Success") + global future_connection_success + future_connection_success.set_result(lifecycle_connect_success_data) + + +if __name__ == '__main__': + print("\nStarting MQTT5 pkcs11 connect Sample\n") + + client = cmdUtils.build_pkcs11_mqtt5_client( + on_lifecycle_stopped=on_lifecycle_stopped, + on_lifecycle_connection_success=on_lifecycle_connection_success) + print("MQTT5 Client Created") + + if is_ci == False: + print("Connecting to {} with client ID '{}'...".format( + cmdUtils.get_command(cmdUtils.m_cmd_endpoint), cmdUtils.get_command("client_id"))) + else: + print("Connecting to endpoint with client ID") + + client.start() + future_connection_success.result(TIMEOUT) + print("Clint Connected") + + print("Stopping Client") + client.stop() + + future_stopped.result(TIMEOUT) + print("Client Stopped!") From dfd510f85322d2a80bfb000f26aab7e624c035f4 Mon Sep 17 00:00:00 2001 From: Steve Kim Date: Thu, 1 Dec 2022 16:40:53 -0800 Subject: [PATCH 46/61] changed softhsm2 conf file name --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2bd0419b..070858c3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -233,8 +233,8 @@ jobs: - name: run MQTT5 PKCS11 Connect sample run: | mkdir -p /tmp/tokens - export SOFTHSM2_CONF=/tmp/softhsm2.conf - echo "directories.tokendir = /tmp/tokens" > /tmp/softhsm2.conf + export SOFTHSM2_CONF=/tmp/mqtt5_softhsm2.conf + echo "directories.tokendir = /tmp/tokens" > /tmp/mqtt5_softhsm2.conf python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --language Python --sample_file "${{ env.CI_SAMPLES_FOLDER }}/mqtt5_pkcs11_connect.py" --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/PubSub/cert' --sample_secret_private_key 'ci/PubSub/keyp8' --sample_run_softhsm 'true' --sample_arguments '--pkcs11_lib /usr/lib/softhsm/libsofthsm2.so --pin 0000 --token_label my-token --key_label my-key' - name: configure AWS credentials (Custom Authorizer) uses: aws-actions/configure-aws-credentials@v1 From 6a55aef4a8584891d56b1bdcc903621f0772c3b1 Mon Sep 17 00:00:00 2001 From: Steve Kim Date: Thu, 1 Dec 2022 16:47:20 -0800 Subject: [PATCH 47/61] don't regen tokens --- .github/workflows/ci.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 070858c3..1fc6ebac 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -232,9 +232,6 @@ jobs: python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --language Python --sample_file "${{ env.CI_SAMPLES_FOLDER }}/pkcs11_connect.py" --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/PubSub/cert' --sample_secret_private_key 'ci/PubSub/keyp8' --sample_run_softhsm 'true' --sample_arguments '--pkcs11_lib /usr/lib/softhsm/libsofthsm2.so --pin 0000 --token_label my-token --key_label my-key' - name: run MQTT5 PKCS11 Connect sample run: | - mkdir -p /tmp/tokens - export SOFTHSM2_CONF=/tmp/mqtt5_softhsm2.conf - echo "directories.tokendir = /tmp/tokens" > /tmp/mqtt5_softhsm2.conf python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --language Python --sample_file "${{ env.CI_SAMPLES_FOLDER }}/mqtt5_pkcs11_connect.py" --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/PubSub/cert' --sample_secret_private_key 'ci/PubSub/keyp8' --sample_run_softhsm 'true' --sample_arguments '--pkcs11_lib /usr/lib/softhsm/libsofthsm2.so --pin 0000 --token_label my-token --key_label my-key' - name: configure AWS credentials (Custom Authorizer) uses: aws-actions/configure-aws-credentials@v1 From 24e176c62e1205ffd7496635482bf3438c144c44 Mon Sep 17 00:00:00 2001 From: Steve Kim Date: Thu, 1 Dec 2022 16:48:29 -0800 Subject: [PATCH 48/61] fix sample --- samples/mqtt5_pkcs11_connect.py | 1 - 1 file changed, 1 deletion(-) diff --git a/samples/mqtt5_pkcs11_connect.py b/samples/mqtt5_pkcs11_connect.py index 15488460..bba0c5e6 100644 --- a/samples/mqtt5_pkcs11_connect.py +++ b/samples/mqtt5_pkcs11_connect.py @@ -16,7 +16,6 @@ cmdUtils.add_common_mqtt5_commands() cmdUtils.add_common_proxy_commands() cmdUtils.add_common_logging_commands() -cmdUtils.register_command("key", "", "Path to your key in PEM format.", True, str) cmdUtils.register_command("cert", "", "Path to your client certificate in PEM format.", True, str) cmdUtils.register_command( "port", From 675964ce174895610f19f72da2f00590c2db748c Mon Sep 17 00:00:00 2001 From: Steve Kim Date: Thu, 1 Dec 2022 16:51:26 -0800 Subject: [PATCH 49/61] restore pkcs11 generation --- .github/workflows/ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1fc6ebac..2bd0419b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -232,6 +232,9 @@ jobs: python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --language Python --sample_file "${{ env.CI_SAMPLES_FOLDER }}/pkcs11_connect.py" --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/PubSub/cert' --sample_secret_private_key 'ci/PubSub/keyp8' --sample_run_softhsm 'true' --sample_arguments '--pkcs11_lib /usr/lib/softhsm/libsofthsm2.so --pin 0000 --token_label my-token --key_label my-key' - name: run MQTT5 PKCS11 Connect sample run: | + mkdir -p /tmp/tokens + export SOFTHSM2_CONF=/tmp/softhsm2.conf + echo "directories.tokendir = /tmp/tokens" > /tmp/softhsm2.conf python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --language Python --sample_file "${{ env.CI_SAMPLES_FOLDER }}/mqtt5_pkcs11_connect.py" --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/PubSub/cert' --sample_secret_private_key 'ci/PubSub/keyp8' --sample_run_softhsm 'true' --sample_arguments '--pkcs11_lib /usr/lib/softhsm/libsofthsm2.so --pin 0000 --token_label my-token --key_label my-key' - name: configure AWS credentials (Custom Authorizer) uses: aws-actions/configure-aws-credentials@v1 From bc1ee1e5126002a5b8a339ec9c875c783d3d9dbf Mon Sep 17 00:00:00 2001 From: Steve Kim Date: Thu, 1 Dec 2022 17:04:42 -0800 Subject: [PATCH 50/61] reversed pkcs11 test order --- .github/workflows/ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2bd0419b..077b83c7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -224,18 +224,18 @@ jobs: - name: run MQTT5 PubSub sample run: | python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --language Python --sample_file "${{ env.CI_SAMPLES_FOLDER }}/mqtt5_pubsub.py" --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/PubSub/cert' --sample_secret_private_key 'ci/PubSub/key' - - name: run PKCS11 Connect sample + - name: run MQTT5 PKCS11 Connect sample run: | mkdir -p /tmp/tokens export SOFTHSM2_CONF=/tmp/softhsm2.conf echo "directories.tokendir = /tmp/tokens" > /tmp/softhsm2.conf - python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --language Python --sample_file "${{ env.CI_SAMPLES_FOLDER }}/pkcs11_connect.py" --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/PubSub/cert' --sample_secret_private_key 'ci/PubSub/keyp8' --sample_run_softhsm 'true' --sample_arguments '--pkcs11_lib /usr/lib/softhsm/libsofthsm2.so --pin 0000 --token_label my-token --key_label my-key' - - name: run MQTT5 PKCS11 Connect sample + python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --language Python --sample_file "${{ env.CI_SAMPLES_FOLDER }}/mqtt5_pkcs11_connect.py" --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/PubSub/cert' --sample_secret_private_key 'ci/PubSub/keyp8' --sample_run_softhsm 'true' --sample_arguments '--pkcs11_lib /usr/lib/softhsm/libsofthsm2.so --pin 0000 --token_label my-token --key_label my-key' + - name: run PKCS11 Connect sample run: | mkdir -p /tmp/tokens export SOFTHSM2_CONF=/tmp/softhsm2.conf echo "directories.tokendir = /tmp/tokens" > /tmp/softhsm2.conf - python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --language Python --sample_file "${{ env.CI_SAMPLES_FOLDER }}/mqtt5_pkcs11_connect.py" --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/PubSub/cert' --sample_secret_private_key 'ci/PubSub/keyp8' --sample_run_softhsm 'true' --sample_arguments '--pkcs11_lib /usr/lib/softhsm/libsofthsm2.so --pin 0000 --token_label my-token --key_label my-key' + python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --language Python --sample_file "${{ env.CI_SAMPLES_FOLDER }}/pkcs11_connect.py" --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/PubSub/cert' --sample_secret_private_key 'ci/PubSub/keyp8' --sample_run_softhsm 'true' --sample_arguments '--pkcs11_lib /usr/lib/softhsm/libsofthsm2.so --pin 0000 --token_label my-token --key_label my-key' - name: configure AWS credentials (Custom Authorizer) uses: aws-actions/configure-aws-credentials@v1 with: From 5c4714dfd6b4594bb34be32c44df1cf423277057 Mon Sep 17 00:00:00 2001 From: Steve Kim Date: Thu, 1 Dec 2022 17:09:50 -0800 Subject: [PATCH 51/61] tmp2 for all second pkcs11 tests --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 077b83c7..37fb083c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -232,9 +232,9 @@ jobs: python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --language Python --sample_file "${{ env.CI_SAMPLES_FOLDER }}/mqtt5_pkcs11_connect.py" --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/PubSub/cert' --sample_secret_private_key 'ci/PubSub/keyp8' --sample_run_softhsm 'true' --sample_arguments '--pkcs11_lib /usr/lib/softhsm/libsofthsm2.so --pin 0000 --token_label my-token --key_label my-key' - name: run PKCS11 Connect sample run: | - mkdir -p /tmp/tokens - export SOFTHSM2_CONF=/tmp/softhsm2.conf - echo "directories.tokendir = /tmp/tokens" > /tmp/softhsm2.conf + mkdir -p /tmp2/tokens + export SOFTHSM2_CONF=/tmp2/softhsm2.conf + echo "directories.tokendir = /tmp2/tokens" > /tmp2/softhsm2.conf python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --language Python --sample_file "${{ env.CI_SAMPLES_FOLDER }}/pkcs11_connect.py" --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/PubSub/cert' --sample_secret_private_key 'ci/PubSub/keyp8' --sample_run_softhsm 'true' --sample_arguments '--pkcs11_lib /usr/lib/softhsm/libsofthsm2.so --pin 0000 --token_label my-token --key_label my-key' - name: configure AWS credentials (Custom Authorizer) uses: aws-actions/configure-aws-credentials@v1 From bad3550195fdfee264bfe1bc1bdc918636641647 Mon Sep 17 00:00:00 2001 From: Steve Kim Date: Fri, 2 Dec 2022 09:06:00 -0800 Subject: [PATCH 52/61] dif conf file --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 37fb083c..907498c4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -232,9 +232,9 @@ jobs: python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --language Python --sample_file "${{ env.CI_SAMPLES_FOLDER }}/mqtt5_pkcs11_connect.py" --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/PubSub/cert' --sample_secret_private_key 'ci/PubSub/keyp8' --sample_run_softhsm 'true' --sample_arguments '--pkcs11_lib /usr/lib/softhsm/libsofthsm2.so --pin 0000 --token_label my-token --key_label my-key' - name: run PKCS11 Connect sample run: | - mkdir -p /tmp2/tokens - export SOFTHSM2_CONF=/tmp2/softhsm2.conf - echo "directories.tokendir = /tmp2/tokens" > /tmp2/softhsm2.conf + mkdir -p /tmp/tokens + export SOFTHSM2_CONF=/tmp/softhsm2-2.conf + echo "directories.tokendir = /tmp/tokens" > /tmp/softhsm2-2.conf python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --language Python --sample_file "${{ env.CI_SAMPLES_FOLDER }}/pkcs11_connect.py" --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/PubSub/cert' --sample_secret_private_key 'ci/PubSub/keyp8' --sample_run_softhsm 'true' --sample_arguments '--pkcs11_lib /usr/lib/softhsm/libsofthsm2.so --pin 0000 --token_label my-token --key_label my-key' - name: configure AWS credentials (Custom Authorizer) uses: aws-actions/configure-aws-credentials@v1 From 5fc5e74caec135548a54df5ed7b51256776aa06d Mon Sep 17 00:00:00 2001 From: Steve Kim Date: Fri, 2 Dec 2022 09:12:29 -0800 Subject: [PATCH 53/61] removed token steps --- .github/workflows/ci.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 907498c4..4d509dd7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -232,9 +232,6 @@ jobs: python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --language Python --sample_file "${{ env.CI_SAMPLES_FOLDER }}/mqtt5_pkcs11_connect.py" --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/PubSub/cert' --sample_secret_private_key 'ci/PubSub/keyp8' --sample_run_softhsm 'true' --sample_arguments '--pkcs11_lib /usr/lib/softhsm/libsofthsm2.so --pin 0000 --token_label my-token --key_label my-key' - name: run PKCS11 Connect sample run: | - mkdir -p /tmp/tokens - export SOFTHSM2_CONF=/tmp/softhsm2-2.conf - echo "directories.tokendir = /tmp/tokens" > /tmp/softhsm2-2.conf python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --language Python --sample_file "${{ env.CI_SAMPLES_FOLDER }}/pkcs11_connect.py" --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/PubSub/cert' --sample_secret_private_key 'ci/PubSub/keyp8' --sample_run_softhsm 'true' --sample_arguments '--pkcs11_lib /usr/lib/softhsm/libsofthsm2.so --pin 0000 --token_label my-token --key_label my-key' - name: configure AWS credentials (Custom Authorizer) uses: aws-actions/configure-aws-credentials@v1 From 1977d076e3ea663e12e7bc129994219e032f899b Mon Sep 17 00:00:00 2001 From: Steve Kim Date: Fri, 2 Dec 2022 09:15:57 -0800 Subject: [PATCH 54/61] pkcs11 --- .github/workflows/ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4d509dd7..077b83c7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -232,6 +232,9 @@ jobs: python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --language Python --sample_file "${{ env.CI_SAMPLES_FOLDER }}/mqtt5_pkcs11_connect.py" --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/PubSub/cert' --sample_secret_private_key 'ci/PubSub/keyp8' --sample_run_softhsm 'true' --sample_arguments '--pkcs11_lib /usr/lib/softhsm/libsofthsm2.so --pin 0000 --token_label my-token --key_label my-key' - name: run PKCS11 Connect sample run: | + mkdir -p /tmp/tokens + export SOFTHSM2_CONF=/tmp/softhsm2.conf + echo "directories.tokendir = /tmp/tokens" > /tmp/softhsm2.conf python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --language Python --sample_file "${{ env.CI_SAMPLES_FOLDER }}/pkcs11_connect.py" --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/PubSub/cert' --sample_secret_private_key 'ci/PubSub/keyp8' --sample_run_softhsm 'true' --sample_arguments '--pkcs11_lib /usr/lib/softhsm/libsofthsm2.so --pin 0000 --token_label my-token --key_label my-key' - name: configure AWS credentials (Custom Authorizer) uses: aws-actions/configure-aws-credentials@v1 From 196fd76ed1d9356d4bca19ee3ec08007f75f7fe6 Mon Sep 17 00:00:00 2001 From: Steve Kim Date: Fri, 2 Dec 2022 09:35:11 -0800 Subject: [PATCH 55/61] run both samples together --- .github/workflows/ci.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 077b83c7..5aa6dea1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -230,11 +230,6 @@ jobs: export SOFTHSM2_CONF=/tmp/softhsm2.conf echo "directories.tokendir = /tmp/tokens" > /tmp/softhsm2.conf python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --language Python --sample_file "${{ env.CI_SAMPLES_FOLDER }}/mqtt5_pkcs11_connect.py" --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/PubSub/cert' --sample_secret_private_key 'ci/PubSub/keyp8' --sample_run_softhsm 'true' --sample_arguments '--pkcs11_lib /usr/lib/softhsm/libsofthsm2.so --pin 0000 --token_label my-token --key_label my-key' - - name: run PKCS11 Connect sample - run: | - mkdir -p /tmp/tokens - export SOFTHSM2_CONF=/tmp/softhsm2.conf - echo "directories.tokendir = /tmp/tokens" > /tmp/softhsm2.conf python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --language Python --sample_file "${{ env.CI_SAMPLES_FOLDER }}/pkcs11_connect.py" --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/PubSub/cert' --sample_secret_private_key 'ci/PubSub/keyp8' --sample_run_softhsm 'true' --sample_arguments '--pkcs11_lib /usr/lib/softhsm/libsofthsm2.so --pin 0000 --token_label my-token --key_label my-key' - name: configure AWS credentials (Custom Authorizer) uses: aws-actions/configure-aws-credentials@v1 From 2c9bac858bf71ff4b2dca1810cbb4a0ef2107824 Mon Sep 17 00:00:00 2001 From: Steve Kim Date: Fri, 2 Dec 2022 09:42:28 -0800 Subject: [PATCH 56/61] changed labels --- .github/workflows/ci.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5aa6dea1..76ffa3ca 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -230,7 +230,12 @@ jobs: export SOFTHSM2_CONF=/tmp/softhsm2.conf echo "directories.tokendir = /tmp/tokens" > /tmp/softhsm2.conf python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --language Python --sample_file "${{ env.CI_SAMPLES_FOLDER }}/mqtt5_pkcs11_connect.py" --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/PubSub/cert' --sample_secret_private_key 'ci/PubSub/keyp8' --sample_run_softhsm 'true' --sample_arguments '--pkcs11_lib /usr/lib/softhsm/libsofthsm2.so --pin 0000 --token_label my-token --key_label my-key' - python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --language Python --sample_file "${{ env.CI_SAMPLES_FOLDER }}/pkcs11_connect.py" --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/PubSub/cert' --sample_secret_private_key 'ci/PubSub/keyp8' --sample_run_softhsm 'true' --sample_arguments '--pkcs11_lib /usr/lib/softhsm/libsofthsm2.so --pin 0000 --token_label my-token --key_label my-key' + - name: run PKCS11 Connect sample + run: | + mkdir -p /tmp/tokens + export SOFTHSM2_CONF=/tmp/softhsm2.conf + echo "directories.tokendir = /tmp/tokens" > /tmp/softhsm2.conf + python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --language Python --sample_file "${{ env.CI_SAMPLES_FOLDER }}/pkcs11_connect.py" --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/PubSub/cert' --sample_secret_private_key 'ci/PubSub/keyp8' --sample_run_softhsm 'true' --sample_arguments '--pkcs11_lib /usr/lib/softhsm/libsofthsm2.so --pin 0000 --token_label my-token-2 --key_label my-key-2' - name: configure AWS credentials (Custom Authorizer) uses: aws-actions/configure-aws-credentials@v1 with: From d9508462f03a6fc323fbdefd6e6c2116747edffe Mon Sep 17 00:00:00 2001 From: Steve Kim Date: Fri, 2 Dec 2022 10:04:28 -0800 Subject: [PATCH 57/61] changed pkcs11 token tmp folder --- .github/workflows/ci.yml | 38 +++++++++++++++++++++++++++++++++++--- 1 file changed, 35 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 76ffa3ca..c2b7241d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -226,16 +226,16 @@ jobs: python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --language Python --sample_file "${{ env.CI_SAMPLES_FOLDER }}/mqtt5_pubsub.py" --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/PubSub/cert' --sample_secret_private_key 'ci/PubSub/key' - name: run MQTT5 PKCS11 Connect sample run: | - mkdir -p /tmp/tokens + mkdir -p /tmp/mqtt5/tokens export SOFTHSM2_CONF=/tmp/softhsm2.conf - echo "directories.tokendir = /tmp/tokens" > /tmp/softhsm2.conf + echo "directories.tokendir = /tmp/mqtt5/tokens" > /tmp/softhsm2.conf python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --language Python --sample_file "${{ env.CI_SAMPLES_FOLDER }}/mqtt5_pkcs11_connect.py" --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/PubSub/cert' --sample_secret_private_key 'ci/PubSub/keyp8' --sample_run_softhsm 'true' --sample_arguments '--pkcs11_lib /usr/lib/softhsm/libsofthsm2.so --pin 0000 --token_label my-token --key_label my-key' - name: run PKCS11 Connect sample run: | mkdir -p /tmp/tokens export SOFTHSM2_CONF=/tmp/softhsm2.conf echo "directories.tokendir = /tmp/tokens" > /tmp/softhsm2.conf - python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --language Python --sample_file "${{ env.CI_SAMPLES_FOLDER }}/pkcs11_connect.py" --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/PubSub/cert' --sample_secret_private_key 'ci/PubSub/keyp8' --sample_run_softhsm 'true' --sample_arguments '--pkcs11_lib /usr/lib/softhsm/libsofthsm2.so --pin 0000 --token_label my-token-2 --key_label my-key-2' + python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --language Python --sample_file "${{ env.CI_SAMPLES_FOLDER }}/pkcs11_connect.py" --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/PubSub/cert' --sample_secret_private_key 'ci/PubSub/keyp8' --sample_run_softhsm 'true' --sample_arguments '--pkcs11_lib /usr/lib/softhsm/libsofthsm2.so --pin 0000 --token_label my-token --key_label my-key' - name: configure AWS credentials (Custom Authorizer) uses: aws-actions/configure-aws-credentials@v1 with: @@ -272,6 +272,38 @@ jobs: python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --language Python --sample_file "${{ env.CI_SAMPLES_FOLDER }}/fleetprovisioning.py" --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/FleetProvisioning/cert' --sample_secret_private_key 'ci/FleetProvisioning/key' --sample_arguments "--template_name CI_FleetProvisioning_Template --template_parameters {\"SerialNumber\":\"${Sample_UUID}\"}" python3 ${{ env.CI_UTILS_FOLDER }}/delete_iot_thing_ci.py --thing_name "Fleet_Thing_${Sample_UUID}" --region "us-east-1" + # Runs the samples and ensures that everything is working + linux-smoke-tests-mqtt5: + runs-on: ubuntu-latest + permissions: + id-token: write # This is required for requesting the JWT + steps: + - name: Build ${{ env.PACKAGE_NAME }} + run: | + python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')" + chmod a+x builder + ./builder build -p ${{ env.PACKAGE_NAME }} + - name: Running samples in CI setup + run: | + python3 -m pip install boto3 + sudo apt-get update -y + sudo apt-get install softhsm -y + softhsm2-util --version + - name: configure AWS credentials (Connect and PubSub) + uses: aws-actions/configure-aws-credentials@v1 + with: + role-to-assume: ${{ env.CI_PUBSUB_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} + - name: run MQTT5 PubSub sample + run: | + python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --language Python --sample_file "${{ env.CI_SAMPLES_FOLDER }}/mqtt5_pubsub.py" --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/PubSub/cert' --sample_secret_private_key 'ci/PubSub/key' + - name: run MQTT5 PKCS11 Connect sample + run: | + mkdir -p /tmp/tokens + export SOFTHSM2_CONF=/tmp/softhsm2.conf + echo "directories.tokendir = /tmp/tokens" > /tmp/softhsm2.conf + python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --language Python --sample_file "${{ env.CI_SAMPLES_FOLDER }}/mqtt5_pkcs11_connect.py" --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/PubSub/cert' --sample_secret_private_key 'ci/PubSub/keyp8' --sample_run_softhsm 'true' --sample_arguments '--pkcs11_lib /usr/lib/softhsm/libsofthsm2.so --pin 0000 --token_label my-token --key_label my-key' + # check that docs can still build check-docs: runs-on: ubuntu-20.04 # latest From 604bc2d16fff8a629d46def0a0fd5906dc892a1f Mon Sep 17 00:00:00 2001 From: Steve Kim Date: Fri, 2 Dec 2022 10:10:25 -0800 Subject: [PATCH 58/61] remove mqtt5 specific job --- .github/workflows/ci.yml | 32 -------------------------------- 1 file changed, 32 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c2b7241d..b2625e1f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -272,38 +272,6 @@ jobs: python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --language Python --sample_file "${{ env.CI_SAMPLES_FOLDER }}/fleetprovisioning.py" --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/FleetProvisioning/cert' --sample_secret_private_key 'ci/FleetProvisioning/key' --sample_arguments "--template_name CI_FleetProvisioning_Template --template_parameters {\"SerialNumber\":\"${Sample_UUID}\"}" python3 ${{ env.CI_UTILS_FOLDER }}/delete_iot_thing_ci.py --thing_name "Fleet_Thing_${Sample_UUID}" --region "us-east-1" - # Runs the samples and ensures that everything is working - linux-smoke-tests-mqtt5: - runs-on: ubuntu-latest - permissions: - id-token: write # This is required for requesting the JWT - steps: - - name: Build ${{ env.PACKAGE_NAME }} - run: | - python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')" - chmod a+x builder - ./builder build -p ${{ env.PACKAGE_NAME }} - - name: Running samples in CI setup - run: | - python3 -m pip install boto3 - sudo apt-get update -y - sudo apt-get install softhsm -y - softhsm2-util --version - - name: configure AWS credentials (Connect and PubSub) - uses: aws-actions/configure-aws-credentials@v1 - with: - role-to-assume: ${{ env.CI_PUBSUB_ROLE }} - aws-region: ${{ env.AWS_DEFAULT_REGION }} - - name: run MQTT5 PubSub sample - run: | - python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --language Python --sample_file "${{ env.CI_SAMPLES_FOLDER }}/mqtt5_pubsub.py" --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/PubSub/cert' --sample_secret_private_key 'ci/PubSub/key' - - name: run MQTT5 PKCS11 Connect sample - run: | - mkdir -p /tmp/tokens - export SOFTHSM2_CONF=/tmp/softhsm2.conf - echo "directories.tokendir = /tmp/tokens" > /tmp/softhsm2.conf - python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --language Python --sample_file "${{ env.CI_SAMPLES_FOLDER }}/mqtt5_pkcs11_connect.py" --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/PubSub/cert' --sample_secret_private_key 'ci/PubSub/keyp8' --sample_run_softhsm 'true' --sample_arguments '--pkcs11_lib /usr/lib/softhsm/libsofthsm2.so --pin 0000 --token_label my-token --key_label my-key' - # check that docs can still build check-docs: runs-on: ubuntu-20.04 # latest From 1bad3f5cf9503034db25b267d6c0521acc36d6b2 Mon Sep 17 00:00:00 2001 From: Steve Kim Date: Fri, 2 Dec 2022 11:00:03 -0800 Subject: [PATCH 59/61] custom authorizer test/sample added --- .github/workflows/ci.yml | 3 + samples/mqtt5_custom_authorizer_connect.py | 70 ++++++++++++++++++++++ samples/mqtt5_pkcs11_connect.py | 10 +--- 3 files changed, 76 insertions(+), 7 deletions(-) create mode 100644 samples/mqtt5_custom_authorizer_connect.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b2625e1f..e051a4ef 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -244,6 +244,9 @@ jobs: - name: run CustomAuthorizerConnect sample run: | python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --language Python --sample_file "${{ env.CI_SAMPLES_FOLDER }}/custom_authorizer_connect.py" --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_custom_authorizer_name 'ci/CustomAuthorizer/name' --sample_secret_custom_authorizer_password 'ci/CustomAuthorizer/password' + - name: run MQTT5 CustomAuthorizerConnect sample + run: | + python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --language Python --sample_file "${{ env.CI_SAMPLES_FOLDER }}/mqtt5_custom_authorizer_connect.py" --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_custom_authorizer_name 'ci/CustomAuthorizer/name' --sample_secret_custom_authorizer_password 'ci/CustomAuthorizer/password' - name: configure AWS credentials (Shadow) uses: aws-actions/configure-aws-credentials@v1 with: diff --git a/samples/mqtt5_custom_authorizer_connect.py b/samples/mqtt5_custom_authorizer_connect.py new file mode 100644 index 00000000..282c846d --- /dev/null +++ b/samples/mqtt5_custom_authorizer_connect.py @@ -0,0 +1,70 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0. + +from awsiot import mqtt5_client_builder +from awscrt import mqtt5 +from uuid import uuid4 +from concurrent.futures import Future + +TIMEOUT = 100 + +# Parse arguments +import command_line_utils +cmdUtils = command_line_utils.CommandLineUtils( + "Custom Authorizer Connect - Make a MQTT5 Client connection using a custom authorizer.") +cmdUtils.add_common_mqtt_commands() +cmdUtils.add_common_logging_commands() +cmdUtils.add_common_custom_authorizer_commands() +cmdUtils.register_command("client_id", "", + "Client ID to use for MQTT connection (optional, default='test-*').", + default="test-" + str(uuid4())) +cmdUtils.register_command("is_ci", "", "If present the sample will run in CI mode (optional, default='None')") +# Needs to be called so the command utils parse the commands +cmdUtils.get_args() + +future_stopped = Future() +future_connection_success = Future() +is_ci = cmdUtils.get_command("is_ci", None) != None + +# Callback for the lifecycle event Stopped +def on_lifecycle_stopped(lifecycle_stopped_data: mqtt5.LifecycleStoppedData): + print("Lifecycle Stopped") + global future_stopped + future_stopped.set_result(lifecycle_stopped_data) + + +# Callback for the lifecycle event Connection Success +def on_lifecycle_connection_success(lifecycle_connect_success_data: mqtt5.LifecycleConnectSuccessData): + print("Lifecycle Connection Success") + global future_connection_success + future_connection_success.set_result(lifecycle_connect_success_data) + +if __name__ == '__main__': + + # Create MQTT5 Client with a custom authorizer + client = mqtt5_client_builder.direct_with_custom_authorizer( + endpoint=cmdUtils.get_command_required(cmdUtils.m_cmd_endpoint), + ca_filepath=cmdUtils.get_command(cmdUtils.m_cmd_ca_file), + auth_username=cmdUtils.get_command(cmdUtils.m_cmd_custom_auth_username), + auth_authorizer_name=cmdUtils.get_command(cmdUtils.m_cmd_custom_auth_authorizer_name), + auth_authorizer_signature=cmdUtils.get_command(cmdUtils.m_cmd_custom_auth_authorizer_signature), + auth_password=cmdUtils.get_command(cmdUtils.m_cmd_custom_auth_password), + on_lifecycle_stopped=on_lifecycle_stopped, + on_lifecycle_connection_success=on_lifecycle_connection_success, + client_id=cmdUtils.get_command("client_id")) + + if is_ci == False: + print("Connecting to {} with client ID '{}'...".format( + cmdUtils.get_command(cmdUtils.m_cmd_endpoint), cmdUtils.get_command("client_id"))) + else: + print("Connecting to endpoint with client ID") + + client.start() + future_connection_success.result(TIMEOUT) + print("Clint Connected") + + print("Stopping Client") + client.stop() + + future_stopped.result(TIMEOUT) + print("Client Stopped!") diff --git a/samples/mqtt5_pkcs11_connect.py b/samples/mqtt5_pkcs11_connect.py index bba0c5e6..7087e2f8 100644 --- a/samples/mqtt5_pkcs11_connect.py +++ b/samples/mqtt5_pkcs11_connect.py @@ -2,17 +2,14 @@ # SPDX-License-Identifier: Apache-2.0. import command_line_utils -from awscrt import mqtt5, exceptions +from awscrt import mqtt5 from uuid import uuid4 -import threading from concurrent.futures import Future -import time TIMEOUT = 100 -topic_filter = "test/topic" # Parse arguments -cmdUtils = command_line_utils.CommandLineUtils("PubSub - Send and receive messages through an MQTT5 connection.") +cmdUtils = command_line_utils.CommandLineUtils("MQTT5 PKCS11 Connect - Make a MQTT5 Client connection using PKCS11.") cmdUtils.add_common_mqtt5_commands() cmdUtils.add_common_proxy_commands() cmdUtils.add_common_logging_commands() @@ -40,8 +37,6 @@ future_connection_success = Future() is_ci = cmdUtils.get_command("is_ci", None) != None -# Callback when any publish is received - # Callback for the lifecycle event Stopped def on_lifecycle_stopped(lifecycle_stopped_data: mqtt5.LifecycleStoppedData): print("Lifecycle Stopped") @@ -59,6 +54,7 @@ def on_lifecycle_connection_success(lifecycle_connect_success_data: mqtt5.Lifecy if __name__ == '__main__': print("\nStarting MQTT5 pkcs11 connect Sample\n") + # Create MQTT5 Client with using PKCS11 client = cmdUtils.build_pkcs11_mqtt5_client( on_lifecycle_stopped=on_lifecycle_stopped, on_lifecycle_connection_success=on_lifecycle_connection_success) From 5ac19573be7808262bab8deeeaa71ea2a626f581 Mon Sep 17 00:00:00 2001 From: Steve Kim Date: Fri, 2 Dec 2022 11:58:38 -0800 Subject: [PATCH 60/61] update MQTT5 Guide --- documents/MQTT5.md | 133 +++++++++++++++++++++++++++++++++------------ 1 file changed, 97 insertions(+), 36 deletions(-) diff --git a/documents/MQTT5.md b/documents/MQTT5.md index 5fab7739..b80b5b49 100644 --- a/documents/MQTT5.md +++ b/documents/MQTT5.md @@ -1,10 +1,33 @@ # MQTT 5 -## Developer Preview Disclaimer +# Table of Contents + +* [Developer Preview Disclaimer](#developer-preview-disclaimer) +* [Introduction](#introduction) +* [MQTT5 differences relative to MQTT311 implementation](#mqtt5-differences-relative-to-mqtt311-implementation) + * [Major Changes](#major-changes) + * [Minor Changes](#minor-changes) + * [Not Supported](#not-supported) +* [Getting Started with MQTT5](#getting-started-with-mqtt5) + * [Connecting to AWS IoT Core](#connecting-to-aws-iot-core) + * [How to create a MQTT5 Client based on desired connection method](#how-to-create-a-mqtt5-client-based-on-desired-connection-method) + * [Direct MQTT with X509-based mutual TLS](#direct-mqtt-with-x509-based-mutual-tls) + * [MQTT over Websockets with Sigv4 authentication](#mqtt-over-websockets-with-sigv4-authentication) + * [Direct MQTT with Custom Authentication](#direct-mqtt-with-custom-authentication) + * [Direct MQTT with PKCS11 Method](#direct-mqtt-with-pkcs11-method) + * [HTTP Proxy](#http-proxy) + * [Client Lifecycle Management](#client-lifecycle-management) + * [Lifecycle Events](#lifecycle-events) + * [Client Operations](#client-operations) + * [Subscribe](#subscribe) + * [Unsubscribe](#unsubscribe) + * [Publish](#publish) + * [MQTT5 Best Practices](#mqtt5-best-practices) +## **Developer Preview Disclaimer** MQTT5 support is currently in **developer preview**. We encourage feedback at all times, but feedback during the preview window is especially valuable in shaping the final product. During the preview period we may make backwards-incompatible changes to the public API, but in general, this is something we will try our best to avoid. The MQTT5 client cannot yet be used with the AWS IoT MQTT services (Shadow, Jobs, Identity). This is a shortcoming that we hope to address in the near future. -## Introduction +## **Introduction** This user guide is designed to act as a reference and guide for how to use MQTT5 with the Java SDK. This guide includes code snippets for how to make a MQTT5 client with proper configuration, how to connect to AWS IoT Core, how to perform operations and interact with AWS IoT Core through MQTT5, and some best practices for MQTT5. @@ -18,7 +41,7 @@ If you are completely new to MQTT, it is highly recommended to check out the fol This user guide expects some beginner level familiarity with MQTT and the terms used to describe MQTT. -## What's Different? (relative to the MQTT311 implementation) +## **MQTT5 differences relative to MQTT311 implementation** SDK MQTT5 support comes from a separate client implementation. In doing so, we took the opportunity to incorporate feedback about the 311 client that we could not apply without making breaking changes. If you're used to the 311 client's API contract, there are a number of differences. ### Major changes @@ -42,40 +65,16 @@ Not all parts of the MQTT5 spec are supported by the implementation. We current * AUTH packets and the authentication fields in the CONNECT packet * QoS 2 -## Client lifecycle management -Once created, an MQTT5 client's configuration is immutable. Invoking start() on the client will put it into an active state where it -recurrently establishes a connection to the configured remote endpoint. Reconnecting continues until you invoke stop(). - -``` -# Create an MQTT5 Client - - client_options = mqtt5.ClientOptions( - host_name = , - port = ) +## **Getting Started with MQTT5** - # Other options in client options can be set but once Client is initialized configuration is immutable - # e.g. setting the on_publish_callback_fn to be called - # client_options.on_publish_callback_fn = on_publish_received - - client = mqtt5.Client(client_options) +This section covers how to use MQTT5 in the Python SDK. This includes how to setup a MQTT5 builder for making MQTT5 clients, how to connect to AWS IoT Core, and how to perform the operations with the MQTT5 client. Each section below contains code snippets showing the functionality in Python. - -# Use the client - client.start(); - ... -``` - -Invoking stop() breaks the current connection (if any) and moves the client into an idle state. - -``` - // Shutdown - client.stop(); - -``` - -## Connecting To AWS IoT Core +## **Connecting To AWS IoT Core** We strongly recommend using the AwsIotMqtt5ClientConfigBuilder class to configure MQTT5 clients when connecting to AWS IoT Core. The builder simplifies configuration for all authentication methods supported by AWS IoT Core. This section shows samples for all of the authentication possibilities. +## **How to create a MQTT5 Client based on desired connection method** +### **Optional Keyword Arguments** +All lifecycle events and the callback for publishes received by the MQTT5 Client should be added to the builder on creation of the Client. A full list of accepted arguments can be found in the API guide. #### **Direct MQTT with X509-based mutual TLS** For X509 based mutual TLS, you can create a client where the certificate and private key are configured by path: ``` @@ -141,6 +140,29 @@ If your custom authorizer uses signing, you must specify the three signed token ``` In both cases, the builder will construct a final CONNECT packet username field value for you based on the values configured. Do not add the token-signing fields to the value of the username that you assign within the custom authentication config structure. Similarly, do not add any custom authentication related values to the username in the CONNECT configuration optionally attached to the client configuration. The builder will do everything for you. +#### **Direct MQTT with PKCS11 Method** + +A MQTT5 direct connection can be made using a PKCS11 device rather than using a PEM encoded private key, the private key for mutual TLS is stored on a PKCS#11 compatible smart card or Hardware Security Module (HSM). To create a MQTT5 builder configured for this connection, see the following code: + +``` + # other builder configurations can be added using **kwargs in the builder + + pkcs11_lib = io.Pkcs11Lib( + file=, + behavior=io.Pkcs11Lib.InitializeFinalizeBehavior.STRICT) + + client = mqtt5_client_builder.mtls_with_pkcs11( + pkcs11_lib=pkcs11_lib, + user_pin=user_pin, + slot_id=pkcs11_slot_id, + token_label=pkcs11_token_label, + priave_key_label=pkcs11_private_key_label, + cert_filepath=pkcs11_cert_filepath, + endpoint = ) +``` + +**Note**: Currently, TLS integration with PKCS#11 is only available on Unix devices. + #### **HTTP Proxy** No matter what your connection transport or authentication method is, you may connect through an HTTP proxy by adding the http_proxy_options keyword argument to the builder: @@ -159,7 +181,37 @@ by adding the http_proxy_options keyword argument to the builder: SDK Proxy support also includes support for basic authentication and TLS-to-proxy. SDK proxy support does not include any additional proxy authentication methods (kerberos, NTLM, etc...) nor does it include non-HTTP proxies (SOCKS5, for example). -## Lifecycle Events +## **Client lifecycle management** +Once created, an MQTT5 client's configuration is immutable. Invoking start() on the client will put it into an active state where it +recurrently establishes a connection to the configured remote endpoint. Reconnecting continues until you invoke stop(). + +``` +# Create an MQTT5 Client + + client_options = mqtt5.ClientOptions( + host_name = , + port = ) + + # Other options in client options can be set but once Client is initialized configuration is immutable + # e.g. setting the on_publish_callback_fn to be called + # client_options.on_publish_callback_fn = on_publish_received + + client = mqtt5.Client(client_options) + + +# Use the client + client.start(); + ... +``` + +Invoking stop() breaks the current connection (if any) and moves the client into an idle state. + +``` + // Shutdown + client.stop(); + +``` +## **Lifecycle Events** The MQTT5 client emits a set of events related to state and network status changes. #### **AttemptingConnect** @@ -177,7 +229,7 @@ Emitted when the client's network connection is shut down, either by a local act #### **Stopped** Emitted once the client has shutdown any associated network connection and entered an idle state where it will no longer attempt to reconnect. Only emitted after an invocation of stop() on the client. A stopped client may always be started again. -## Client Operations +## **Client Operations** There are four basic MQTT operations you can perform with the MQTT5 client. ### Subscribe @@ -215,4 +267,13 @@ The stop() API supports a DISCONNECT packet as an optional parameter. If suppli client.stop(mqtt5.DisconnectPacket( reason_code = mqtt5.DisconnectReasonCode.NORMAL_DISCONNECTION, session_expiry_interval_sec = 3600)) -``` \ No newline at end of file +``` + +## **MQTT5 Best Practices** + +Below are some best practices for the MQTT5 client that are recommended to follow for the best development experience: + +* When creating MQTT5 clients, make sure to use ClientIDs that are unique! If you connect two MQTT5 clients with the same ClientID, they will Disconnect each other! If you do not configure a ClientID, the MQTT5 server will automatically assign one. +* Use the minimum QoS you can get away with for the lowest latency and bandwidth costs. For example, if you are sending data consistently multiple times per second and do not have to have a guarantee the server got each and every publish, using QoS 0 may be ideal compared to QoS 1. Of course, this heavily depends on your use case but generally it is recommended to use the lowest QoS possible. +* If you are getting unexpected disconnects when trying to connect to AWS IoT Core, make sure to check your IoT Core Thing’s policy and permissions to make sure your device is has the permissions it needs to connect! +* For **Publish**, **Subscribe**, and **Unsubscribe**, you can check the reason codes in the returned Future to see if the operation actually succeeded. \ No newline at end of file From 177526d02ab14ed82e7c6f25329586e83ba52386 Mon Sep 17 00:00:00 2001 From: Steve Kim Date: Fri, 2 Dec 2022 12:05:15 -0800 Subject: [PATCH 61/61] updated guides and pr fixes --- samples/README.md | 8 +++++++- samples/mqtt5_pubsub.py | 11 +++++------ 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/samples/README.md b/samples/README.md index 765d18f0..7454d0ba 100644 --- a/samples/README.md +++ b/samples/README.md @@ -27,7 +27,13 @@ python3 pubsub.py --help This sample uses the [Message Broker](https://docs.aws.amazon.com/iot/latest/developerguide/iot-message-broker.html) for AWS IoT to send and receive messages -through an MQTT5 connection. On startup, the device connects to the server, +through an MQTT5 connection. + +MQTT5 introduces additional features and enhancements that improve the development experience with MQTT. You can read more about MQTT5 in the Python V2 SDK by checking out the [MQTT5 user guide](../documents/MQTT5.md). + +Note: MQTT5 support is currently in **developer preview**. We encourage feedback at all times, but feedback during the preview window is especially valuable in shaping the final product. During the preview period we may make backwards-incompatible changes to the public API, but in general, this is something we will try our best to avoid. + +On startup, the device connects to the server, subscribes to a topic, and begins publishing messages to that topic. The device should receive those same messages back from the message broker, since it is subscribed to that same topic. diff --git a/samples/mqtt5_pubsub.py b/samples/mqtt5_pubsub.py index f2888c6c..2f649b2d 100644 --- a/samples/mqtt5_pubsub.py +++ b/samples/mqtt5_pubsub.py @@ -46,8 +46,6 @@ is_ci = cmdUtils.get_command("is_ci", None) != None # Callback when any publish is received - - def on_publish_received(publish_packet_data): publish_packet = publish_packet_data.publish_packet assert isinstance(publish_packet, mqtt5.PublishPacket) @@ -101,10 +99,11 @@ def on_lifecycle_connection_failure(lifecycle_connection_failure: mqtt5.Lifecycl lifecycle_connect_success_data = future_connection_success.result(TIMEOUT) connack_packet = lifecycle_connect_success_data.connack_packet negotiated_settings = lifecycle_connect_success_data.negotiated_settings - print("Connected to endpoint:'{}' with Client ID:'{}' with reason_code:{}".format( - cmdUtils.get_command(cmdUtils.m_cmd_endpoint), - connack_packet.assigned_client_identifier, - repr(connack_packet.reason_code))) + if is_ci == False: + print("Connected to endpoint:'{}' with Client ID:'{}' with reason_code:{}".format( + cmdUtils.get_command(cmdUtils.m_cmd_endpoint), + connack_packet.assigned_client_identifier, + repr(connack_packet.reason_code))) # Subscribe