From 590b02490add4a59094fd330469a67d4e0e6f465 Mon Sep 17 00:00:00 2001 From: Noah Beard Date: Thu, 9 Mar 2023 13:40:09 -0500 Subject: [PATCH 01/12] Initial work on operation statistics queue sample --- samples/operation_queue/command_line_utils.py | 403 ++++++++++++++++ .../operation_queue/mqtt_operation_queue.py | 431 ++++++++++++++++++ .../mqtt_operation_queue_tests.py | 0 samples/operation_queue/operation_queue.py | 169 +++++++ 4 files changed, 1003 insertions(+) create mode 100644 samples/operation_queue/command_line_utils.py create mode 100644 samples/operation_queue/mqtt_operation_queue.py create mode 100644 samples/operation_queue/mqtt_operation_queue_tests.py create mode 100644 samples/operation_queue/operation_queue.py diff --git a/samples/operation_queue/command_line_utils.py b/samples/operation_queue/command_line_utils.py new file mode 100644 index 00000000..dc8af5c2 --- /dev/null +++ b/samples/operation_queue/command_line_utils.py @@ -0,0 +1,403 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0. + +import argparse +from awscrt import io, http, auth +from awsiot import mqtt_connection_builder, mqtt5_client_builder + + +class CommandLineUtils: + def __init__(self, description) -> None: + self.parser = argparse.ArgumentParser(description="Send and receive messages through and MQTT connection.") + self.commands = {} + self.parsed_commands = None + + def register_command(self, command_name, example_input, help_output, required=False, type=None, default=None, choices=None, action=None): + self.commands[command_name] = { + "name":command_name, + "example_input":example_input, + "help_output":help_output, + "required": required, + "type": type, + "default": default, + "choices": choices, + "action": action + } + + def remove_command(self, command_name): + if command_name in self.commands.keys(): + self.commands.pop(command_name) + + def get_args(self): + # if we have already parsed, then return the cached parsed commands + if self.parsed_commands is not None: + return self.parsed_commands + + # add all the commands + for command in self.commands.values(): + if not command["action"] is None: + self.parser.add_argument("--" + command["name"], action=command["action"], help=command["help_output"], + required=command["required"], default=command["default"]) + else: + self.parser.add_argument("--" + command["name"], metavar=command["example_input"], help=command["help_output"], + required=command["required"], type=command["type"], default=command["default"], choices=command["choices"]) + + self.parsed_commands = self.parser.parse_args() + # Automatically start logging if it is set + if self.parsed_commands.verbosity: + io.init_logging(getattr(io.LogLevel, self.parsed_commands.verbosity), 'stderr') + return self.parsed_commands + + def update_command(self, command_name, new_example_input=None, new_help_output=None, new_required=None, new_type=None, new_default=None, new_action=None): + if command_name in self.commands.keys(): + if new_example_input: + self.commands[command_name]["example_input"] = new_example_input + if new_help_output: + self.commands[command_name]["help_output"] = new_help_output + if new_required: + self.commands[command_name]["required"] = new_required + if new_type: + self.commands[command_name]["type"] = new_type + if new_default: + self.commands[command_name]["default"] = new_default + if new_action: + self.commands[command_name]["action"] = new_action + + def add_common_mqtt_commands(self): + self.register_command( + self.m_cmd_endpoint, + "", + "The endpoint of the mqtt server not including a port.", + True, + str) + self.register_command( + self.m_cmd_ca_file, + "", + "Path to AmazonRootCA1.pem (optional, system trust store used by default)", + False, + str) + + def add_common_mqtt5_commands(self): + self.register_command( + self.m_cmd_endpoint, + "", + "The endpoint of the mqtt server not including a port.", + True, + str) + self.register_command( + self.m_cmd_ca_file, + "", + "Path to AmazonRootCA1.pem (optional, system trust store used by default)", + False, + str) + + def add_common_proxy_commands(self): + self.register_command( + self.m_cmd_proxy_host, + "", + "Host name of the proxy server to connect through (optional)", + False, + str) + self.register_command( + self.m_cmd_proxy_port, + "", + "Port of the http proxy to use (optional, default='8080')", + type=int, + default=8080) + + def add_common_topic_message_commands(self): + self.register_command( + self.m_cmd_topic, + "", + "Topic to publish, subscribe to (optional, default='test/topic').", + default="test/topic") + self.register_command( + self.m_cmd_message, + "", + "The message to send in the payload (optional, default='Hello World!').", + default="Hello World!") + + def add_common_logging_commands(self): + self.register_command( + self.m_cmd_verbosity, + "", + "Logging level.", + default=io.LogLevel.NoLogs.name, + choices=[ + x.name for x in io.LogLevel]) + + def add_common_custom_authorizer_commands(self): + self.register_command( + self.m_cmd_custom_auth_username, + "", + "The name to send when connecting through the custom authorizer (optional)") + self.register_command( + self.m_cmd_custom_auth_authorizer_name, + "", + "The name of the custom authorizer to connect to (optional but required for everything but custom domains)") + self.register_command( + self.m_cmd_custom_auth_username, + "", + "The signature to send when connecting through a custom authorizer (optional)") + self.register_command( + self.m_cmd_custom_auth_password, + "", + "The password to send when connecting through a custom authorizer (optional)") + + """ + Returns the command if it exists and has been passed to the console, otherwise it will print the help for the sample and exit the application. + """ + def get_command_required(self, command_name, message=None): + if hasattr(self.parsed_commands, command_name): + return getattr(self.parsed_commands, command_name) + else: + self.parser.print_help() + print("Command --" + command_name + " required.") + if message is not None: + print(message) + exit() + + """ + Returns the command if it exists and has been passed to the console, otherwise it returns whatever is passed as the default. + """ + def get_command(self, command_name, default=None): + if hasattr(self.parsed_commands, command_name): + return getattr(self.parsed_commands, command_name) + return default + + def build_pkcs11_mqtt_connection(self, on_connection_interrupted, on_connection_resumed): + + pkcs11_lib_path = self.get_command_required(self.m_cmd_pkcs11_lib) + print(f"Loading PKCS#11 library '{pkcs11_lib_path}' ...") + pkcs11_lib = io.Pkcs11Lib( + file=pkcs11_lib_path, + behavior=io.Pkcs11Lib.InitializeFinalizeBehavior.STRICT) + print("Loaded!") + + pkcs11_slot_id = None + if (self.get_command(self.m_cmd_pkcs11_slot) != None): + pkcs11_slot_id = int(self.get_command(self.m_cmd_pkcs11_slot)) + + # Create MQTT connection + mqtt_connection = mqtt_connection_builder.mtls_with_pkcs11( + pkcs11_lib=pkcs11_lib, + user_pin=self.get_command_required(self.m_cmd_pkcs11_pin), + slot_id=pkcs11_slot_id, + token_label=self.get_command_required(self.m_cmd_pkcs11_token), + private_key_label=self.get_command_required(self.m_cmd_pkcs11_key), + cert_filepath=self.get_command_required(self.m_cmd_pkcs11_cert), + endpoint=self.get_command_required(self.m_cmd_endpoint), + port=self.get_command("port"), + ca_filepath=self.get_command(self.m_cmd_ca_file), + on_connection_interrupted=on_connection_interrupted, + on_connection_resumed=on_connection_resumed, + client_id=self.get_command_required("client_id"), + clean_session=False, + keep_alive_secs=30) + return mqtt_connection + + def build_websocket_mqtt_connection(self, on_connection_interrupted, on_connection_resumed): + proxy_options = self.get_proxy_options_for_mqtt_connection() + credentials_provider = auth.AwsCredentialsProvider.new_default_chain() + mqtt_connection = mqtt_connection_builder.websockets_with_default_aws_signing( + endpoint=self.get_command_required(self.m_cmd_endpoint), + region=self.get_command_required(self.m_cmd_signing_region), + credentials_provider=credentials_provider, + http_proxy_options=proxy_options, + ca_filepath=self.get_command(self.m_cmd_ca_file), + on_connection_interrupted=on_connection_interrupted, + on_connection_resumed=on_connection_resumed, + client_id=self.get_command_required("client_id"), + clean_session=False, + keep_alive_secs=30) + return mqtt_connection + + def build_cognito_mqtt_connection(self, on_connection_interrupted, on_connection_resumed): + proxy_options = self.get_proxy_options_for_mqtt_connection() + + cognito_endpoint = "cognito-identity." + self.get_command_required(self.m_cmd_signing_region) + ".amazonaws.com" + credentials_provider = auth.AwsCredentialsProvider.new_cognito( + endpoint=cognito_endpoint, + identity=self.get_command_required(self.m_cmd_cognito_identity), + tls_ctx=io.ClientTlsContext(io.TlsContextOptions())) + + mqtt_connection = mqtt_connection_builder.websockets_with_default_aws_signing( + endpoint=self.get_command_required(self.m_cmd_endpoint), + region=self.get_command_required(self.m_cmd_signing_region), + credentials_provider=credentials_provider, + http_proxy_options=proxy_options, + ca_filepath=self.get_command(self.m_cmd_ca_file), + on_connection_interrupted=on_connection_interrupted, + on_connection_resumed=on_connection_resumed, + client_id=self.get_command_required("client_id"), + clean_session=False, + keep_alive_secs=30) + return mqtt_connection + + def build_direct_mqtt_connection(self, on_connection_interrupted, on_connection_resumed): + proxy_options = self.get_proxy_options_for_mqtt_connection() + mqtt_connection = mqtt_connection_builder.mtls_from_path( + endpoint=self.get_command_required(self.m_cmd_endpoint), + port=self.get_command_required("port"), + cert_filepath=self.get_command_required(self.m_cmd_cert_file), + pri_key_filepath=self.get_command_required(self.m_cmd_key_file), + ca_filepath=self.get_command(self.m_cmd_ca_file), + on_connection_interrupted=on_connection_interrupted, + on_connection_resumed=on_connection_resumed, + client_id=self.get_command_required("client_id"), + clean_session=False, + keep_alive_secs=30, + http_proxy_options=proxy_options) + return mqtt_connection + + def build_mqtt_connection(self, on_connection_interrupted, on_connection_resumed): + if self.get_command(self.m_cmd_signing_region) is not None: + return self.build_websocket_mqtt_connection(on_connection_interrupted, on_connection_resumed) + else: + return self.build_direct_mqtt_connection(on_connection_interrupted, on_connection_resumed) + + def get_proxy_options_for_mqtt_connection(self): + proxy_options = None + if self.parsed_commands.proxy_host and self.parsed_commands.proxy_port: + proxy_options = http.HttpProxyOptions( + host_name=self.parsed_commands.proxy_host, + port=self.parsed_commands.proxy_port) + return proxy_options + + ######################################################################## + # MQTT5 + ######################################################################## + + def build_pkcs11_mqtt5_client(self, + on_publish_received=None, + on_lifecycle_stopped=None, + on_lifecycle_attempting_connect=None, + on_lifecycle_connection_success=None, + on_lifecycle_connection_failure=None, + on_lifecycle_disconnection=None): + + pkcs11_lib_path = self.get_command_required(self.m_cmd_pkcs11_lib) + print(f"Loading PKCS#11 library '{pkcs11_lib_path}' ...") + pkcs11_lib = io.Pkcs11Lib( + file=pkcs11_lib_path, + behavior=io.Pkcs11Lib.InitializeFinalizeBehavior.STRICT) + print("Loaded!") + + pkcs11_slot_id = None + if (self.get_command(self.m_cmd_pkcs11_slot) is not None): + pkcs11_slot_id = int(self.get_command(self.m_cmd_pkcs11_slot)) + + # Create MQTT5 client + mqtt5_client = mqtt5_client_builder.mtls_with_pkcs11( + pkcs11_lib=pkcs11_lib, + user_pin=self.get_command_required(self.m_cmd_pkcs11_pin), + slot_id=pkcs11_slot_id, + token_label=self.get_command_required(self.m_cmd_pkcs11_token), + private_key_label=self.get_command_required(self.m_cmd_pkcs11_key), + cert_filepath=self.get_command_required(self.m_cmd_pkcs11_cert), + endpoint=self.get_command_required(self.m_cmd_endpoint), + port=self.get_command("port"), + ca_filepath=self.get_command(self.m_cmd_ca_file), + on_publish_received=on_publish_received, + on_lifecycle_stopped=on_lifecycle_stopped, + on_lifecycle_attempting_connect=on_lifecycle_attempting_connect, + on_lifecycle_connection_success=on_lifecycle_connection_success, + on_lifecycle_connection_failure=on_lifecycle_connection_failure, + on_lifecycle_disconnection=on_lifecycle_disconnection, + client_id=self.get_command("client_id")) + + return mqtt5_client + + def build_websocket_mqtt5_client(self, + on_publish_received=None, + on_lifecycle_stopped=None, + on_lifecycle_attempting_connect=None, + on_lifecycle_connection_success=None, + on_lifecycle_connection_failure=None, + on_lifecycle_disconnection=None): + proxy_options = self.get_proxy_options_for_mqtt_connection() + credentials_provider = auth.AwsCredentialsProvider.new_default_chain() + mqtt5_client = mqtt5_client_builder.websockets_with_default_aws_signing( + endpoint=self.get_command_required(self.m_cmd_endpoint), + region=self.get_command_required(self.m_cmd_signing_region), + credentials_provider=credentials_provider, + http_proxy_options=proxy_options, + ca_filepath=self.get_command(self.m_cmd_ca_file), + on_publish_received=on_publish_received, + on_lifecycle_stopped=on_lifecycle_stopped, + on_lifecycle_attempting_connect=on_lifecycle_attempting_connect, + on_lifecycle_connection_success=on_lifecycle_connection_success, + on_lifecycle_connection_failure=on_lifecycle_connection_failure, + on_lifecycle_disconnection=on_lifecycle_disconnection, + client_id=self.get_command_required("client_id")) + return mqtt5_client + + def build_direct_mqtt5_client(self, + on_publish_received=None, + on_lifecycle_stopped=None, + on_lifecycle_attempting_connect=None, + on_lifecycle_connection_success=None, + on_lifecycle_connection_failure=None, + on_lifecycle_disconnection=None): + proxy_options = self.get_proxy_options_for_mqtt_connection() + mqtt5_client = mqtt5_client_builder.mtls_from_path( + endpoint=self.get_command_required(self.m_cmd_endpoint), + port=self.get_command_required("port"), + cert_filepath=self.get_command_required(self.m_cmd_cert_file), + pri_key_filepath=self.get_command_required(self.m_cmd_key_file), + ca_filepath=self.get_command(self.m_cmd_ca_file), + http_proxy_options=proxy_options, + on_publish_received=on_publish_received, + on_lifecycle_stopped=on_lifecycle_stopped, + on_lifecycle_attempting_connect=on_lifecycle_attempting_connect, + on_lifecycle_connection_success=on_lifecycle_connection_success, + on_lifecycle_connection_failure=on_lifecycle_connection_failure, + on_lifecycle_disconnection=on_lifecycle_disconnection, + client_id=self.get_command_required("client_id")) + return mqtt5_client + + def build_mqtt5_client(self, + on_publish_received=None, + on_lifecycle_stopped=None, + on_lifecycle_attempting_connect=None, + on_lifecycle_connection_success=None, + on_lifecycle_connection_failure=None, + on_lifecycle_disconnection=None): + + if self.get_command(self.m_cmd_signing_region) is not None: + return self.build_websocket_mqtt5_client(on_publish_received=on_publish_received, + on_lifecycle_stopped=on_lifecycle_stopped, + on_lifecycle_attempting_connect=on_lifecycle_attempting_connect, + on_lifecycle_connection_success=on_lifecycle_connection_success, + on_lifecycle_connection_failure=on_lifecycle_connection_failure, + on_lifecycle_disconnection=on_lifecycle_disconnection) + else: + return self.build_direct_mqtt5_client(on_publish_received=on_publish_received, + on_lifecycle_stopped=on_lifecycle_stopped, + on_lifecycle_attempting_connect=on_lifecycle_attempting_connect, + on_lifecycle_connection_success=on_lifecycle_connection_success, + on_lifecycle_connection_failure=on_lifecycle_connection_failure, + on_lifecycle_disconnection=on_lifecycle_disconnection) + + # Constants for commonly used/needed commands + m_cmd_endpoint = "endpoint" + m_cmd_ca_file = "ca_file" + m_cmd_cert_file = "cert" + m_cmd_key_file = "key" + m_cmd_proxy_host = "proxy_host" + m_cmd_proxy_port = "proxy_port" + m_cmd_signing_region = "signing_region" + m_cmd_pkcs11_lib = "pkcs11_lib" + m_cmd_pkcs11_cert = "cert" + m_cmd_pkcs11_pin = "pin" + m_cmd_pkcs11_token = "token_label" + m_cmd_pkcs11_slot = "slot_id" + m_cmd_pkcs11_key = "key_label" + m_cmd_message = "message" + m_cmd_topic = "topic" + m_cmd_verbosity = "verbosity" + m_cmd_custom_auth_username = "custom_auth_username" + m_cmd_custom_auth_authorizer_name = "custom_auth_authorizer_name" + m_cmd_custom_auth_authorizer_signature = "custom_auth_authorizer_signature" + m_cmd_custom_auth_password = "custom_auth_password" + m_cmd_cognito_identity = "cognito_identity" diff --git a/samples/operation_queue/mqtt_operation_queue.py b/samples/operation_queue/mqtt_operation_queue.py new file mode 100644 index 00000000..5737e12a --- /dev/null +++ b/samples/operation_queue/mqtt_operation_queue.py @@ -0,0 +1,431 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0. + +from threading import Thread, Lock +from enum import Enum +from time import sleep +from types import FunctionType + +from awscrt import mqtt + +####################################################### +# Enums +######################################################## + +class QueueResult(Enum): + SUCCESS=0 + ERROR_QUEUE_FULL=1 + ERROR_INVALID_ARGUMENT=2 + UNKNOWN_QUEUE_LIMIT_BEHAVIOR=3 + UNKNOWN_QUEUE_INSERT_BEHAVIOR=4 + UNKNOWN_OPERATION=5 + UNKNOWN_ERROR=6 + +class QueueOperationType(Enum): + NONE=0 + PUBLISH=1 + SUBSCRIBE=2 + UNSUBSCRIBE=3 + +class LimitBehavior(Enum): + DROP_FRONT=0 + DROP_BACK=1 + RETURN_ERROR=2 + +class InsertBehavior(Enum): + INSERT_FRONT=0 + INSERT_BACK=1 + +class QueueOperation(): + type : QueueOperationType = QueueOperationType.NONE + topic : str = "" + payload: any = None + qos : mqtt.QoS = mqtt.QoS.AT_MOST_ONCE + retain : bool = None + subscribe_callback : FunctionType = None + +####################################################### +# Classes +######################################################## + +class OperationQueueBuilder: + _connection : mqtt.Connection = None + _queue_limit_size : int = 10 + _queue_limit_behavior : LimitBehavior = LimitBehavior.DROP_BACK + _queue_insert_behavior : InsertBehavior = InsertBehavior.INSERT_BACK + _incomplete_limit : int = 1 + _inflight_limit : int = 1 + _on_operation_sent_callback : FunctionType = None + _on_operation_sent_failure_callback : FunctionType = None + _on_operation_dropped_callback : FunctionType = None + _on_queue_full_callback : FunctionType = None + _on_queue_empty_callback : FunctionType = None + _queue_loop_time_ms : int = 1000 + _enable_logging : bool = False + + def with_connection(self, connection:mqtt.Connection): + self._connection = connection + return self + + def get_connection(self) -> mqtt.Connection: + return self._connection + + def with_queue_limit_size(self, queue_limit_size:int): + self._queue_limit_size = queue_limit_size + return self + + def get_connection(self) -> int: + return self._queue_limit_size + + def with_queue_limit_behavior(self, queue_limit_behavior:LimitBehavior): + self._queue_limit_behavior = queue_limit_behavior + return self + + def get_queue_limit_behavior(self) -> LimitBehavior: + return self._queue_limit_behavior + + def with_queue_insert_behavior(self, queue_insert_behavior:InsertBehavior): + self._queue_insert_behavior = queue_insert_behavior + return self + + def get_queue_insert_behavior(self) -> InsertBehavior: + return self._queue_insert_behavior + + def with_incomplete_limit(self, incomplete_limit:int): + self._incomplete_limit = incomplete_limit + return self + + def get_incomplete_limit(self) -> int: + return self._incomplete_limit + + def with_inflight_limit(self, inflight_limit:int): + self._inflight_limit = inflight_limit + return self + + def get_inflight_limit(self) -> int: + return self._inflight_limit + + def with_on_operation_sent_callback(self, on_operation_sent_callback: FunctionType): + self._on_operation_sent_callback = on_operation_sent_callback + return self + + def get_on_operation_sent_callback(self) -> FunctionType: + return self._on_operation_sent_callback + + def with_on_operation_sent_failure_callback(self, on_operation_sent_failure_callback: FunctionType): + self._on_operation_sent_failure_callback = on_operation_sent_failure_callback + return self + + def get_on_operation_sent_failure_callback(self) -> FunctionType: + return self._on_operation_sent_failure_callback + + def with_on_operation_dropped_callback(self, on_operation_dropped_callback: FunctionType): + self._on_operation_dropped_callback = on_operation_dropped_callback + return self + + def get_on_operation_dropped_callback(self) -> FunctionType: + return self._on_operation_dropped_callback + + def with_on_queue_full_callback(self, on_queue_full_callback:FunctionType): + self._on_queue_full_callback = on_queue_full_callback + return self + + def get_on_queue_full_callback(self) -> FunctionType: + return self._on_queue_full_callback + + def with_on_queue_empty_callback(self, on_queue_empty_callback:FunctionType): + self._on_queue_empty_callback = on_queue_empty_callback + return self + + def get_on_queue_empty_callback(self) -> FunctionType: + return self._on_queue_empty_callback + + def with_queue_loop_time(self, queue_loop_time_ms:int): + self._queue_loop_time_ms = queue_loop_time_ms + return self + + def get_queue_loop_time(self) -> int: + return self._queue_loop_time_ms + + def with_enable_logging(self, enable_logging:bool): + self._enable_logging = enable_logging + return self + + def get_enable_logging(self) -> bool: + return self._enable_logging + + def build(self): + return OperationQueue(self) + + +class OperationQueue: + _operation_queue : list[QueueOperation] = [] + _operation_queue_lock : Lock = Lock() + _operation_queue_thread : Thread = None + _operation_queue_thread_running : bool = False + # configuration options/settings + _connection : mqtt.Connection = None + _queue_limit_size : int = 10 + _queue_limit_behavior : LimitBehavior = LimitBehavior.DROP_BACK + _queue_insert_behavior : InsertBehavior = InsertBehavior.INSERT_BACK + _incomplete_limit : int = 1 + _inflight_limit : int = 1 + _on_operation_sent_callback : FunctionType = None + _on_operation_sent_failure_callback : FunctionType = None + _on_operation_dropped_callback : FunctionType = None + _on_queue_full_callback : FunctionType = None + _on_queue_empty_callback : FunctionType = None + _queue_loop_time_ms : int = 1000 + _enable_logging : bool = False + + def __init__(self, builder:OperationQueueBuilder) -> None: + self._connection = builder._connection + self._queue_limit_size = builder._queue_limit_size + self._queue_limit_behavior = builder._queue_limit_behavior + self._queue_insert_behavior = builder._queue_insert_behavior + self._incomplete_limit = builder._incomplete_limit + self._inflight_limit = builder._inflight_limit + self._on_operation_sent_callback = builder._on_operation_sent_callback + self._on_operation_sent_failure_callback = builder._on_operation_sent_failure_callback + self._on_operation_dropped_callback = builder._on_operation_dropped_callback + self._on_queue_full_callback = builder._on_queue_full_callback + self._on_queue_empty_callback = builder._on_queue_empty_callback + self._queue_loop_time_ms = builder._queue_loop_time_ms + self._enable_logging = builder._enable_logging + + #################### + # HELPER FUNCTIONS + #################### + + def _add_operation_to_queue_insert(self, operation: QueueOperation) -> QueueResult: + result : QueueResult = QueueResult.SUCCESS + if (self._queue_insert_behavior == InsertBehavior.INSERT_FRONT): + self._operation_queue.insert(0, operation) + elif (self._queue_insert_behavior == InsertBehavior.INSERT_BACK): + self._operation_queue.insert(len(self._operation_queue), operation) + else: + result = QueueResult.UNKNOWN_QUEUE_INSERT_BEHAVIOR + return result + + def _add_operation_to_queue_overflow(self, operation: QueueOperation) -> tuple: + result = [QueueResult.SUCCESS, None] + if (self._queue_limit_behavior == LimitBehavior.RETURN_ERROR): + self._print_log_message("Did not drop any operation, instead returning error...") + result[0] = QueueResult.ERROR_QUEUE_FULL + elif (self._queue_limit_behavior == LimitBehavior.DROP_FRONT): + result[1] = self._operation_queue[0] + del self._operation_queue[0] + self._print_log_message(f"Dropped operation of type {result[1].type} from the front...") + result[0] = self._add_operation_to_queue_insert(operation) + elif (self._queue_limit_behavior == LimitBehavior.DROP_BACK): + end_of_queue = len(self._operation_queue)-1 + result[1] = self._operation_queue[end_of_queue] + del self._operation_queue[end_of_queue] + self._print_log_message(f"Dropped operation of type {result[1].type} from the back...") + result[0] = self._add_operation_to_queue_insert(operation) + else: + result[0] = QueueResult.UNKNOWN_QUEUE_LIMIT_BEHAVIOR + return result + + def _add_operation_to_queue(self, operation: QueueOperation) -> QueueResult: + result : QueueResult = QueueResult.SUCCESS + dropped_operation : QueueOperation = None + + if (operation == None): + return QueueResult.ERROR_INVALID_ARGUMENT + + # CRITICAL SECTION + self._operation_queue_lock.acquire() + + try: + if (self._queue_limit_size <= 0): + result = self._add_operation_to_queue_insert(operation) + else: + if (len(self._operation_queue)+1 <= self._queue_limit_size): + result = self._add_operation_to_queue_insert(operation) + else: + return_data = self._add_operation_to_queue_overflow(operation) + dropped_operation = return_data[0] + result = return_data[1] + except Exception as exception: + self._print_log_message(f"Exception ocurred adding operation to queue. Exception: {exception}") + + self._operation_queue_lock.release() + # END CRITICAL SECTION + + if (result == QueueResult.SUCCESS): + self._print_log_message(f"Added operation of type {operation.type} successfully to queue") + if (len(self._operation_queue) == self._queue_limit_size and dropped_operation == None): + if (self._on_queue_full_callback != None): + self._on_queue_full_callback() + + # Note: We invoke the dropped callback outside of the critical section to avoid deadlocks + if (dropped_operation != None): + if (self._on_operation_dropped_callback != None): + self._on_operation_dropped_callback(dropped_operation) + + return result + + def _print_log_message(self, message:str) -> None: + if self._enable_logging == True: + print("[MqttOperationQueue] " + message) + + #################### + # LOOP FUNCTIONS + #################### + + def _perform_operation_publish(self, operation: QueueOperation) -> None: + result = self._connection.publish(operation.topic, operation.payload, operation.qos, operation.retain) + if (self._on_operation_sent_callback != None): + self._on_operation_sent_callback(operation, result) + + def _perform_operation_subscribe(self, operation: QueueOperation) -> None: + result = self._connection.subscribe(operation.topic, operation.qos, operation.subscribe_callback) + if (self._on_operation_sent_callback != None): + self._on_operation_sent_callback(operation, result) + + def _perform_operation_unsubscribe(self, operation: QueueOperation) -> None: + result = self._connection.unsubscribe(operation.topic) + if (self._on_operation_sent_callback != None): + self._on_operation_sent_callback(operation, result) + + def _perform_operation_unknown(self, operation: QueueOperation) -> None: + if (operation == None): + self._print_log_message("ERROR - got empty/none operation to perform") + else: + self._print_log_message("ERROR - got unknown operation to perform") + if (self._on_operation_sent_failure_callback != None): + self._on_operation_sent_failure_callback(operation, QueueResult.UNKNOWN_OPERATION) + + def _perform_operation(self, operation: QueueOperation) -> None: + if (operation == None): + self._perform_operation_unknown(operation) + elif (operation.type == QueueOperationType.PUBLISH): + self._perform_operation_publish(operation) + elif (operation.type == QueueOperationType.SUBSCRIBE): + self._perform_operation_subscribe(operation) + elif (operation.type == QueueOperationType.UNSUBSCRIBE): + self._perform_operation_unsubscribe(operation) + else: + self._perform_operation_unknown(operation) + + def _check_operation_statistics(self) -> bool: + statistics: mqtt.OperationStatisticsData = self._connection.get_stats() + if (statistics.incomplete_operation_count >= self._incomplete_limit): + if (self._incomplete_limit > 0): + self._print_log_message("Skipping running operation due to incomplete operation count being equal or higher than maximum") + return False + if (statistics.unacked_operation_count >= self._inflight_limit): + if (self._inflight_limit > 0): + self._print_log_message("Skipping running operation due to inflight operation count being equal or higher than maximum") + return False + return True + + def _run_operation(self) -> None: + # CRITICAL SECTION + self._operation_queue_lock.acquire() + + try: + if (len(self._operation_queue) > 0): + operation : QueueOperation = self._operation_queue[0] + del self._operation_queue[0] + + self._print_log_message(f"Starting to perform operation of type {operation.type}") + self._perform_operation(operation) + + if (len(self._operation_queue) <= 0): + if (self._on_queue_empty_callback != None): + self._on_queue_empty_callback() + + pass + else: + self._print_log_message("No operations to perform") + except Exception as exception: + self._print_log_message(f"Exception ocurred performing operation! Exception: {exception}") + + self._operation_queue_lock.release() + # END CRITICAL SECTION + + def _queue_loop(self) -> None: + while self._operation_queue_thread_running == True: + self._print_log_message("Performing operation loop...") + if (self._check_operation_statistics()): + self._run_operation() + sleep(self._queue_loop_time_ms / 1000.0) + pass + + #################### + # OPERATIONS + #################### + + def start(self) -> None: + if (self._operation_queue_thread != None): + self._print_log_message("Cannot start because queue is already started!") + return + self._operation_queue_thread = Thread(target=self._queue_loop) + self._operation_queue_thread_running = True + self._operation_queue_thread.start() + self._print_log_message("Started successfully") + + def stop(self) -> None: + if (self._operation_queue_thread == None): + self._print_log_message("Cannot stop because queue is already stopped!") + return + self._operation_queue_thread_running = False + + # wait for the thread to finish + self._print_log_message("Waiting for thread to stop...") + self._operation_queue_thread.join() + self._operation_queue_thread = None + self._print_log_message("Stopped successfully") + + def publish(self, topic, payload, qos, retain=False) -> QueueResult: + new_operation = QueueOperation() + new_operation.type = QueueOperationType.PUBLISH + new_operation.topic = topic + new_operation.payload = payload + new_operation.qos = qos + new_operation.retain = retain + return self._add_operation_to_queue(new_operation) + + def subscribe(self, topic, qos, callback=None) -> QueueResult: + new_operation = QueueOperation() + new_operation.type = QueueOperationType.SUBSCRIBE + new_operation.topic = topic + new_operation.qos = qos + new_operation.subscribe_callback = callback + return self._add_operation_to_queue(new_operation) + + def unsubscribe(self, topic) -> QueueResult: + new_operation = QueueOperation() + new_operation.type = QueueOperationType.UNSUBSCRIBE + new_operation.topic = topic + return self._add_operation_to_queue(new_operation) + + def add_queue_operation(self, operation: QueueOperation) -> QueueResult: + # Basic validation + if (operation.type == QueueOperationType.NONE): + return QueueResult.ERROR_INVALID_ARGUMENT + elif (operation.type == QueueOperationType.PUBLISH): + if (operation.topic == None or operation.qos == None): + return QueueResult.ERROR_INVALID_ARGUMENT + elif (operation.type == QueueOperationType.SUBSCRIBE): + if (operation.topic == None or operation.qos == None): + return QueueResult.ERROR_INVALID_ARGUMENT + elif (operation.type == QueueOperationType.UNSUBSCRIBE): + if (operation.topic == None): + return QueueResult.ERROR_INVALID_ARGUMENT + else: + return QueueResult.UNKNOWN_ERROR + return self._add_operation_to_queue(operation) + + def get_queue_size(self) -> int: + # CRITICAL SECTION + self._operation_queue_lock.acquire() + size : int = len(self._operation_queue) + self._operation_queue_lock.release() + # END CRITICAL SECTION + return size + + def get_queue_limit(self) -> int: + return self._queue_limit_size diff --git a/samples/operation_queue/mqtt_operation_queue_tests.py b/samples/operation_queue/mqtt_operation_queue_tests.py new file mode 100644 index 00000000..e69de29b diff --git a/samples/operation_queue/operation_queue.py b/samples/operation_queue/operation_queue.py new file mode 100644 index 00000000..a3bfcf4c --- /dev/null +++ b/samples/operation_queue/operation_queue.py @@ -0,0 +1,169 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0. + +from awscrt import mqtt +import threading +from uuid import uuid4 +import json +from concurrent.futures import Future + +# This sample uses the Message Broker for AWS IoT to send and receive messages +# through an MQTT connection. On startup, the device connects to the server, +# subscribes to a topic, and begins publishing messages to that topic. +# The device should receive those same messages back from the message broker, +# since it is subscribed to that same topic. + +import mqtt_operation_queue + +# Parse arguments +from command_line_utils import CommandLineUtils +cmdUtils = CommandLineUtils("PubSub - Send and recieve messages through an MQTT connection.") +cmdUtils.add_common_mqtt_commands() +cmdUtils.add_common_topic_message_commands() +cmdUtils.add_common_proxy_commands() +cmdUtils.add_common_logging_commands() +cmdUtils.register_command("key", "", "Path to your key in PEM format.", True, str) +cmdUtils.register_command("cert", "", "Path to your client certificate in PEM format.", True, str) +cmdUtils.register_command("port", "", "Connection port. AWS IoT supports 443 and 8883 (optional, default=auto).", type=int) +cmdUtils.register_command("client_id", "", "Client ID to use for MQTT connection (optional, default='test-*').", default="test-" + str(uuid4())) +cmdUtils.register_command("count", "", "The number of messages to send (optional, default='20').", default=20, type=int) +cmdUtils.register_command("queue_limit", "", "The maximum number of operations for the queue (optional, default='10')", default=10, type=int) +cmdUtils.register_command("queue_mode", "", "The mode for the queue to use (optional, default=0)" + + "\n\t0 = Overflow removes from queue back and new operations are pushed to queue back" + + "\n\t1 = Overflow removes from queue front and new operations are pushed to queue back" + + "\n\t2 = Overflow removes from queue front and new operations are pushed to queue front" + + "\n\t3 = Overflow removes from queue back and new operations are pushed to queue front", + default=10, type=int) +cmdUtils.register_command("run_tests", "", + "If set to True (1 or greater), then the queue tests will be run instead of the sample (optional, default=0)") +cmdUtils.register_command("is_ci", "", "If present the sample will run in CI mode (optional, default='None')") +# Needs to be called so the command utils parse the commands +cmdUtils.get_args() + +received_count = 0 +received_all_event = threading.Event() +is_ci = cmdUtils.get_command("is_ci", None) != None + +# Callback when connection is accidentally lost. +def on_connection_interrupted(connection, error, **kwargs): + print("Connection interrupted. error: {}".format(error)) + + +# Callback when an interrupted connection is re-established. +def on_connection_resumed(connection, return_code, session_present, **kwargs): + print("Connection resumed. return_code: {} session_present: {}".format(return_code, session_present)) + + if return_code == mqtt.ConnectReturnCode.ACCEPTED and not session_present: + print("Session did not persist. Resubscribing to existing topics...") + connection.resubscribe_existing_topics() + + +# Callback when the subscribed topic receives a message +def on_message_received(topic, payload, dup, qos, retain, **kwargs): + print("Received message from topic '{}': {}".format(topic, payload)) + global received_count + received_count += 1 + if received_count == cmdUtils.get_command("queue_limit"): + received_all_event.set() + + +# Callback when the mqtt operation queue is completely empty +queue_empty_future = Future() +def on_queue_empty(): + print ("Queue is completely empty!") + global queue_empty_future + queue_empty_future.set_result(None) + + +if __name__ == '__main__': + mqtt_connection = cmdUtils.build_mqtt_connection(on_connection_interrupted, on_connection_resumed) + + queue_builder = mqtt_operation_queue.OperationQueueBuilder() + queue_builder.with_connection(mqtt_connection).with_queue_limit_size(cmdUtils.get_command("queue_limit")) + queue_builder.with_on_queue_empty_callback(on_queue_empty) + if (cmdUtils.get_command("queue_mode") == 0): + queue_builder.with_queue_insert_behavior(mqtt_operation_queue.InsertBehavior.INSERT_BACK) + queue_builder.with_queue_limit_behavior(mqtt_operation_queue.LimitBehavior.DROP_BACK) + elif (cmdUtils.get_command("queue_mode") == 1): + queue_builder.with_queue_insert_behavior(mqtt_operation_queue.InsertBehavior.INSERT_BACK) + queue_builder.with_queue_limit_behavior(mqtt_operation_queue.LimitBehavior.DROP_FRONT) + elif (cmdUtils.get_command("queue_mode") == 2): + queue_builder.with_queue_insert_behavior(mqtt_operation_queue.InsertBehavior.INSERT_FRONT) + queue_builder.with_queue_limit_behavior(mqtt_operation_queue.LimitBehavior.DROP_FRONT) + elif (cmdUtils.get_command("queue_mode") == 3): + queue_builder.with_queue_insert_behavior(mqtt_operation_queue.InsertBehavior.INSERT_FRONT) + queue_builder.with_queue_limit_behavior(mqtt_operation_queue.LimitBehavior.DROP_BACK) + mqtt_queue = queue_builder.build() + + if is_ci == False: + print("Connecting to {} with client ID '{}'...".format( + cmdUtils.get_command(cmdUtils.m_cmd_endpoint), cmdUtils.get_command("client_id"))) + else: + print("Connecting to endpoint with client ID") + connect_future = mqtt_connection.connect() + + # Future.result() waits until a result is available + connect_future.result() + print("Connected!") + + # Start the queue + mqtt_queue.start() + + message_count = cmdUtils.get_command("count") + message_topic = cmdUtils.get_command(cmdUtils.m_cmd_topic) + message_string = cmdUtils.get_command(cmdUtils.m_cmd_message) + + # Subscribe using the queue + print("Subscribing to topic '{}'...".format(message_topic)) + mqtt_queue.subscribe( + topic=message_topic, + qos=mqtt.QoS.AT_LEAST_ONCE, + callback=on_message_received) + # Wait for the queue to be empty, indicating the subscribe was sent + queue_empty_future.result() + + # Reset the queue empty future + queue_empty_future = Future() + # Publish message to server desired number of times using the operation queue. + # This step is skipped if message is blank. + if message_string and message_count > 0: + print (f"Filling queue with {message_count} message(s)") + + publish_count = 1 + while (publish_count <= message_count) or (message_count == 0): + message = "{} [{}]".format(message_string, publish_count) + # print(f"Publishing message to topic '{message_topic}': {message}") + message_json = json.dumps(message) + mqtt_queue.publish( + topic=message_topic, + payload=message_json, + qos=mqtt.QoS.AT_LEAST_ONCE) + publish_count += 1 + + # wait for the queue to be empty + print ("Waiting for all publishes in queue to be sent...") + queue_empty_future.result() + else: + print ("Skipping sending publishes due to message being blank or message count being zero") + + # Wait for all messages to be received and ACKs to be back from the server + if message_count != 0 and not received_all_event.is_set(): + print("Waiting for all messages to be received...") + received_all_event.wait() + print("{} message(s) received.".format(received_count)) + + # Reset the queue empty future + queue_empty_future = Future() + # Unsubscribe using the operation queue + mqtt_queue.unsubscribe(message_topic) + # Wait for the queue to be empty, indicating the unsubscribe was sent + queue_empty_future.result() + + # Stop the queue + mqtt_queue.stop() + + # Disconnect + print("Disconnecting...") + disconnect_future = mqtt_connection.disconnect() + disconnect_future.result() + print("Disconnected!") From 8165b50d2fcfa45f850782467ece3d6da970acea Mon Sep 17 00:00:00 2001 From: Noah Beard Date: Thu, 9 Mar 2023 14:35:36 -0500 Subject: [PATCH 02/12] Added documentation to the operation queue --- .../operation_queue/mqtt_operation_queue.py | 341 +++++++++++++++++- samples/operation_queue/operation_queue.py | 4 +- 2 files changed, 336 insertions(+), 9 deletions(-) diff --git a/samples/operation_queue/mqtt_operation_queue.py b/samples/operation_queue/mqtt_operation_queue.py index 5737e12a..64a45333 100644 --- a/samples/operation_queue/mqtt_operation_queue.py +++ b/samples/operation_queue/mqtt_operation_queue.py @@ -13,6 +13,10 @@ ######################################################## class QueueResult(Enum): + """ + The result of attempting to perform an operation on the MqttOperationQueue. + The value indicates either success or what type of issue was encountered. + """ SUCCESS=0 ERROR_QUEUE_FULL=1 ERROR_INVALID_ARGUMENT=2 @@ -22,21 +26,38 @@ class QueueResult(Enum): UNKNOWN_ERROR=6 class QueueOperationType(Enum): + """ + An enum to indicate the type of data the QueueOperation contains. Used + to differentiate between different operations in a common blob object. + """ NONE=0 PUBLISH=1 SUBSCRIBE=2 UNSUBSCRIBE=3 class LimitBehavior(Enum): + """ + An enum to indicate what happens when the MqttOperationQueue is completely full but new + operations are requested to be added to the queue. + """ DROP_FRONT=0 DROP_BACK=1 RETURN_ERROR=2 class InsertBehavior(Enum): + """ + An enum to indicate what happens when the MqttOperationQueue has a new operation it + needs to add to the queue, configuring where the new operation is added. + """ INSERT_FRONT=0 INSERT_BACK=1 class QueueOperation(): + """ + A blob class containing all of the data an operation can possibly possess, as well as + an enum to indicate what type of operation should be stored within. Used to provide + a common base that all operations can be derived from. + """ type : QueueOperationType = QueueOperationType.NONE topic : str = "" payload: any = None @@ -48,7 +69,13 @@ class QueueOperation(): # Classes ######################################################## -class OperationQueueBuilder: +class MqttOperationQueueBuilder: + """ + A builder that contains all of the options of the MqttOperationQueue. + This is where you can configure how the operations queue works prior to making the final + MqttOperationQueue with the build() function. + """ + _connection : mqtt.Connection = None _queue_limit_size : int = 10 _queue_limit_behavior : LimitBehavior = LimitBehavior.DROP_BACK @@ -64,101 +91,268 @@ class OperationQueueBuilder: _enable_logging : bool = False def with_connection(self, connection:mqtt.Connection): + """ + Sets the mqtt.Connection that will be used by the MqttOperationQueue. + This is a REQUIRED argument that has to be set in order for the MqttOperationQueue to function. + + Keyword Args: + connection (mqtt.Connection): The mqtt.Connection that will be used by the MqttOperationQueue. + """ self._connection = connection return self def get_connection(self) -> mqtt.Connection: + """ + Returns the mqtt.Connection used by the MqttOperationQueue + """ return self._connection def with_queue_limit_size(self, queue_limit_size:int): + """ + Sets the maximum size of the operation queue in the MqttOperationQueue. + Default operation queue size is 10. + + If the number of operations exceeds this number, then the queue will be adjusted + based on the queue_limit_behavior. + + Keyword Args: + queue_limit_size (int): The maximum size of the operation queue in the MqttOperationQueue. + """ self._queue_limit_size = queue_limit_size return self def get_connection(self) -> int: + """ + Returns the maximum size of the operation queue in the MqttOperationQueue. + """ return self._queue_limit_size def with_queue_limit_behavior(self, queue_limit_behavior:LimitBehavior): + """ + Sets how the MqttOperationQueue will behave when the operation queue is full but a + new operation is requested to be added to the queue. + The default is DROP_BACK, which will drop the newest (but last to be executed) operation at the back of the queue. + + Keyword Args: + queue_limit_behavior (LimitBehavior): How the MqttOperationQueue will behave when the operation queue is full. + """ self._queue_limit_behavior = queue_limit_behavior return self def get_queue_limit_behavior(self) -> LimitBehavior: + """ + Returns how the MqttOperationQueue will behave when the operation queue is full. + """ return self._queue_limit_behavior def with_queue_insert_behavior(self, queue_insert_behavior:InsertBehavior): + """ + Sets how the MqttOperationQueue will behave when inserting a new operation into the queue. + The default is INSERT_BACK, which will add the new operation to the back (last to be executed) of the queue. + + Keyword Args: + queue_insert_behavior (InsertBehavior): How the MqttOperationQueue will behave when inserting a new operation into the queue. + """ self._queue_insert_behavior = queue_insert_behavior return self def get_queue_insert_behavior(self) -> InsertBehavior: + """ + Returns how the MqttOperationQueue will behave when inserting a new operation into the queue. + """ return self._queue_insert_behavior def with_incomplete_limit(self, incomplete_limit:int): + """ + Sets the maximum number of incomplete operations that the MQTT connection can have before the + MqttOperationQueue will wait for them to be complete. Incomplete operations are those that have been + sent to the mqtt.Connection but have not been fully processed and responded to from the MQTT server/broker. + + Once the maximum number of incomplete operations is met, the MqttOperationQueue will wait until the number + of incomplete operations is below the set maximum. + + Default is set to 1. Set to 0 for no limit. + + Keyword Args: + incomplete_limit (int): The maximum number of incomplete operations before waiting. + """ self._incomplete_limit = incomplete_limit return self def get_incomplete_limit(self) -> int: + """ + Returns the maximum number of incomplete operations before waiting. + """ return self._incomplete_limit def with_inflight_limit(self, inflight_limit:int): + """ + Sets the maximum number of inflight operations that the MQTT connection can have before the + MqttOperationQueue will wait for them to be complete. inflight operations are those that have been + sent to the mqtt.Connection and sent out to the MQTT server/broker, but an acknowledgement from + the MQTT server/broker has not yet been received. + + Once the maximum number of inflight operations is met, the MqttOperationQueue will wait until the number + of inflight operations is below the set maximum. + + Default is set to 1. Set to 0 for no limit. + + Keyword Args: + inflight_limit (int): The maximum number of inflight operations before waiting. + """ self._inflight_limit = inflight_limit return self def get_inflight_limit(self) -> int: + """ + Returns the maximum number of inflight operations before waiting. + """ return self._inflight_limit def with_on_operation_sent_callback(self, on_operation_sent_callback: FunctionType): + """ + Sets the callback that will be invoked when an operation is removed from the queue and sent successfully. + + The callback needs to have the following signature: + callback_name(operation: QueueOperation, operation_result: tuple) + * operation is the operation data that was just successfully sent + * operation_result is a tuple containing the future and the packet ID returned from the MQTT connection + + Keyword Args: + on_operation_sent_callback (FunctionType): The callback to invoke. + """ self._on_operation_sent_callback = on_operation_sent_callback return self def get_on_operation_sent_callback(self) -> FunctionType: + """ + Returns the callback invoked when an operation is removed from the queue and sent successfully. + """ return self._on_operation_sent_callback def with_on_operation_sent_failure_callback(self, on_operation_sent_failure_callback: FunctionType): + """ + Sets the callback that will be invoked when an operation is removed from the queue but failed to send. + + The callback needs to have the following signature: + callback_name(operation: QueueOperation, error: QueueResult) + * operation is the operation data that failed to be sent + * error is a QueueResult containing the error code for why the operation was not successful. + + Keyword Args: + on_operation_sent_failure_callback (FunctionType): The callback to invoke. + """ self._on_operation_sent_failure_callback = on_operation_sent_failure_callback return self def get_on_operation_sent_failure_callback(self) -> FunctionType: + """ + Returns the callback that will be invoked when an operation is removed from the queue but failed to send. + """ return self._on_operation_sent_failure_callback def with_on_operation_dropped_callback(self, on_operation_dropped_callback: FunctionType): + """ + Sets the callback that will be invoked when the operation queue is full, a new operation was added + to the queue, and so an operation had to be removed/dropped from the queue. + + The callback needs to have the following signature: + callback_name(operation: QueueOperation) + * operation is the operation data that was just dropped from the queue + + Keyword Args: + on_operation_dropped_callback (FunctionType): The callback to invoke. + """ self._on_operation_dropped_callback = on_operation_dropped_callback return self def get_on_operation_dropped_callback(self) -> FunctionType: + """ + Returns the callback that will be invoked when an operation is dropped from the queue + """ return self._on_operation_dropped_callback def with_on_queue_full_callback(self, on_queue_full_callback:FunctionType): + """ + Sets the callback that will be invoked when the operation queue is full. + + The callback needs to be a function that takes no arguments: + callback_name() + + Keyword Args: + on_queue_full_callback (FunctionType): The callback to invoke. + """ self._on_queue_full_callback = on_queue_full_callback return self def get_on_queue_full_callback(self) -> FunctionType: + """ + Returns the callback that will be invoked when the operation queue is full. + """ return self._on_queue_full_callback def with_on_queue_empty_callback(self, on_queue_empty_callback:FunctionType): + """ + Sets the callback that will be invoked when the operation queue is completely empty. + + The callback needs to be a function that takes no arguments: + callback_name() + + Keyword Args: + on_queue_empty_callback (FunctionType): The callback to invoke. + """ self._on_queue_empty_callback = on_queue_empty_callback return self def get_on_queue_empty_callback(self) -> FunctionType: + """ + Returns the callback that will be invoked when the operation queue is completely empty. + """ return self._on_queue_empty_callback def with_queue_loop_time(self, queue_loop_time_ms:int): + """ + Sets the interval, in milliseconds, that the MqttOperationQueue will wait before checking the queue and (possibly) + processing an operation based on the statistics and state of the MqttClientConnection assigned to the MqttOperationQueue. + The default is every second. + + Keyword Args: + queue_loop_time_ms (int): The interval, in milliseconds, that the MqttOperationQueue will wait before checking the queue. + """ self._queue_loop_time_ms = queue_loop_time_ms return self def get_queue_loop_time(self) -> int: + """ + Returns the interval, in milliseconds, that the MqttOperationQueue will wait before checking the queue. + """ return self._queue_loop_time_ms def with_enable_logging(self, enable_logging:bool): + """ + Sets whether the MqttOperationQueue will print logging statements to help debug and determine how the + MqttOperationQueue is functioning. + + Keyword Args: + enable_logging (bool): Whether the MqttOperationQueue will print logging statements. + """ self._enable_logging = enable_logging return self def get_enable_logging(self) -> bool: + """ + Returns whether the MqttOperationQueue will print logging statements. + """ return self._enable_logging def build(self): - return OperationQueue(self) + """ + Returns a new MqttOperationQueue with the options set in the builder + """ + return MqttOperationQueue(self) -class OperationQueue: +class MqttOperationQueue: _operation_queue : list[QueueOperation] = [] _operation_queue_lock : Lock = Lock() _operation_queue_thread : Thread = None @@ -175,10 +369,10 @@ class OperationQueue: _on_operation_dropped_callback : FunctionType = None _on_queue_full_callback : FunctionType = None _on_queue_empty_callback : FunctionType = None - _queue_loop_time_ms : int = 1000 + _queue_loop_time_sec : int = 1 _enable_logging : bool = False - def __init__(self, builder:OperationQueueBuilder) -> None: + def __init__(self, builder:MqttOperationQueueBuilder) -> None: self._connection = builder._connection self._queue_limit_size = builder._queue_limit_size self._queue_limit_behavior = builder._queue_limit_behavior @@ -190,7 +384,7 @@ def __init__(self, builder:OperationQueueBuilder) -> None: self._on_operation_dropped_callback = builder._on_operation_dropped_callback self._on_queue_full_callback = builder._on_queue_full_callback self._on_queue_empty_callback = builder._on_queue_empty_callback - self._queue_loop_time_ms = builder._queue_loop_time_ms + self._queue_loop_time_sec = builder._queue_loop_time_ms / 1000.0 # convert to seconds since that is what sleep uses self._enable_logging = builder._enable_logging #################### @@ -198,6 +392,14 @@ def __init__(self, builder:OperationQueueBuilder) -> None: #################### def _add_operation_to_queue_insert(self, operation: QueueOperation) -> QueueResult: + """ + Helper function: Inserts the given QueueOperation into the queue/list directly. + Used to simplify inserting in front/back based on configuration options. + Called by both _add_operation_to_queue and _add_operation_to_queue_overflow. + + Keyword Args: + operation (QueueOperation): The operation to add. + """ result : QueueResult = QueueResult.SUCCESS if (self._queue_insert_behavior == InsertBehavior.INSERT_FRONT): self._operation_queue.insert(0, operation) @@ -208,6 +410,14 @@ def _add_operation_to_queue_insert(self, operation: QueueOperation) -> QueueResu return result def _add_operation_to_queue_overflow(self, operation: QueueOperation) -> tuple: + """ + Helper function: Adds the given QueueOperation to the queue when the queue is full. + Used to make separate the logic for when the queue is full from when it is not yet full. + Called by _add_operation_to_queue. + + Keyword Args: + operation (QueueOperation): The operation to add. + """ result = [QueueResult.SUCCESS, None] if (self._queue_limit_behavior == LimitBehavior.RETURN_ERROR): self._print_log_message("Did not drop any operation, instead returning error...") @@ -228,6 +438,12 @@ def _add_operation_to_queue_overflow(self, operation: QueueOperation) -> tuple: return result def _add_operation_to_queue(self, operation: QueueOperation) -> QueueResult: + """ + Helper function: Adds the given QueueOperation to the queue of operations to be processed. + + Keyword Args: + operation (QueueOperation): The operation to add. + """ result : QueueResult = QueueResult.SUCCESS dropped_operation : QueueOperation = None @@ -267,6 +483,13 @@ def _add_operation_to_queue(self, operation: QueueOperation) -> QueueResult: return result def _print_log_message(self, message:str) -> None: + """ + Helper function: Prints to the console if logging is enabled. + Just makes code a little cleaner and easier to process. + + Keyword Args: + message (str): The message to print. + """ if self._enable_logging == True: print("[MqttOperationQueue] " + message) @@ -275,21 +498,33 @@ def _print_log_message(self, message:str) -> None: #################### def _perform_operation_publish(self, operation: QueueOperation) -> None: + """ + Helper function: Takes the publish operation and passes it to the MQTT connection. + """ result = self._connection.publish(operation.topic, operation.payload, operation.qos, operation.retain) if (self._on_operation_sent_callback != None): self._on_operation_sent_callback(operation, result) def _perform_operation_subscribe(self, operation: QueueOperation) -> None: + """ + Helper function: Takes the subscribe operation and passes it to the MQTT connection. + """ result = self._connection.subscribe(operation.topic, operation.qos, operation.subscribe_callback) if (self._on_operation_sent_callback != None): self._on_operation_sent_callback(operation, result) def _perform_operation_unsubscribe(self, operation: QueueOperation) -> None: + """ + Helper function: Takes the unsubscribe operation and passes it to the MQTT connection. + """ result = self._connection.unsubscribe(operation.topic) if (self._on_operation_sent_callback != None): self._on_operation_sent_callback(operation, result) def _perform_operation_unknown(self, operation: QueueOperation) -> None: + """ + Helper function: Takes the operation if it is unknown and sends it as a failure to the callback. + """ if (operation == None): self._print_log_message("ERROR - got empty/none operation to perform") else: @@ -298,6 +533,9 @@ def _perform_operation_unknown(self, operation: QueueOperation) -> None: self._on_operation_sent_failure_callback(operation, QueueResult.UNKNOWN_OPERATION) def _perform_operation(self, operation: QueueOperation) -> None: + """ + Helper function: Based on the operation type, calls the appropriate helper function. + """ if (operation == None): self._perform_operation_unknown(operation) elif (operation.type == QueueOperationType.PUBLISH): @@ -310,6 +548,12 @@ def _perform_operation(self, operation: QueueOperation) -> None: self._perform_operation_unknown(operation) def _check_operation_statistics(self) -> bool: + """ + Helper function: Checks the MQTT connection operation statistics to see if their values are higher than the maximum + values set in MqttOperationQueue. If the value is higher than the maximum in MqttOperationQueue, then it returns false + so an operation on the queue will not be processed. + Called by the _queue_loop() function + """ statistics: mqtt.OperationStatisticsData = self._connection.get_stats() if (statistics.incomplete_operation_count >= self._incomplete_limit): if (self._incomplete_limit > 0): @@ -322,6 +566,11 @@ def _check_operation_statistics(self) -> bool: return True def _run_operation(self) -> None: + """ + Helper function: Takes the operation off the queue, checks what operation it is, and passes + it to the MQTT connection to be run. + Called by the _queue_loop() function + """ # CRITICAL SECTION self._operation_queue_lock.acquire() @@ -347,11 +596,15 @@ def _run_operation(self) -> None: # END CRITICAL SECTION def _queue_loop(self) -> None: + """ + This function is called every queue_loop_time_ms milliseconds. This is where the logic for handling + the queue resides. + """ while self._operation_queue_thread_running == True: self._print_log_message("Performing operation loop...") if (self._check_operation_statistics()): self._run_operation() - sleep(self._queue_loop_time_ms / 1000.0) + sleep(self._queue_loop_time_sec) pass #################### @@ -359,6 +612,13 @@ def _queue_loop(self) -> None: #################### def start(self) -> None: + """ + Starts the MqttOperationQueue running so it can process the queue. + Every queue_loop_time_ms milliseconds it will check the queue to see if there is at least a single + operation waiting. If there is, it will check the MQTT client statistics to determine if + the MQTT connection has the bandwidth for the next operation (based on incomplete_limit and inflight_limit) + and, if the MQTT connection has bandwidth, will start a next operation from the queue. + """ if (self._operation_queue_thread != None): self._print_log_message("Cannot start because queue is already started!") return @@ -368,6 +628,12 @@ def start(self) -> None: self._print_log_message("Started successfully") def stop(self) -> None: + """ + Stops the MqttOperationQueue from running and processing operations that may be left in the queue. + Once stopped, the MqttOperationQueue can be restarted by calling start() again. + + Note: calling stop() will block the thread temporarily as it waits for the operation queue thread to finish + """ if (self._operation_queue_thread == None): self._print_log_message("Cannot stop because queue is already stopped!") return @@ -380,6 +646,22 @@ def stop(self) -> None: self._print_log_message("Stopped successfully") def publish(self, topic, payload, qos, retain=False) -> QueueResult: + """ + Creates a new Publish operation and adds it to the queue to be run. + + Note that the inputs to this function are exactly the same as the publish() function in + the mqtt.connection, but instead of executing the operation as soon as possible, it + will be added to the queue based on the queue_insert_behavior and processed accordingly. + + The on_operation_sent callback function will be invoked when the operation is + processed and sent by the client. + + Keyword Args: + topic (str): The topic to send the publish to + payload (any): The payload to send + qos (mqtt.QoS): The Quality of Service (QoS) to send the publish with + retain (bool): Whether the publish should be retained on the server + """ new_operation = QueueOperation() new_operation.type = QueueOperationType.PUBLISH new_operation.topic = topic @@ -389,6 +671,21 @@ def publish(self, topic, payload, qos, retain=False) -> QueueResult: return self._add_operation_to_queue(new_operation) def subscribe(self, topic, qos, callback=None) -> QueueResult: + """ + Creates a new subscribe operation and adds it to the queue to be run. + + Note that the inputs to this function are exactly the same as the subscribe() function in + the mqtt.connection, but instead of executing the operation as soon as possible, it + will be added to the queue based on the queue_insert_behavior and processed accordingly. + + The on_operation_sent callback function will be invoked when the operation is + processed and sent by the client. + + Keyword Args: + topic (str): The topic to subscribe to + qos (mqtt.QoS): The Quality of Service (QoS) to send the subscribe with + callback (function): The function to invoke when a publish is sent to the subscribed topic + """ new_operation = QueueOperation() new_operation.type = QueueOperationType.SUBSCRIBE new_operation.topic = topic @@ -397,12 +694,36 @@ def subscribe(self, topic, qos, callback=None) -> QueueResult: return self._add_operation_to_queue(new_operation) def unsubscribe(self, topic) -> QueueResult: + """ + Creates a new unsubscribe operation and adds it to the queue to be run. + + Note that the inputs to this function are exactly the same as the unsubscribe() function in + the mqtt.connection, but instead of executing the operation as soon as possible, it + will be added to the queue based on the queue_insert_behavior and processed accordingly. + + The on_operation_sent callback function will be invoked when the operation is + processed and sent by the client. + + Keyword Args: + topic (str): The topic to unsubscribe to + """ new_operation = QueueOperation() new_operation.type = QueueOperationType.UNSUBSCRIBE new_operation.topic = topic return self._add_operation_to_queue(new_operation) def add_queue_operation(self, operation: QueueOperation) -> QueueResult: + """ + Adds a new queue operation (publish, subscribe, unsubscribe) to the queue to be run. + + Note: This function provides only basic validation of the operation data. It is primarily + intended to be used with the on_operation_dropped callback for when you may want to + add a dropped message back to the queue. + (for example, say it's an important message you know you want to send) + + Keyword Args: + operation (QueueOperation): The operation to add to the queue + """ # Basic validation if (operation.type == QueueOperationType.NONE): return QueueResult.ERROR_INVALID_ARGUMENT @@ -420,6 +741,9 @@ def add_queue_operation(self, operation: QueueOperation) -> QueueResult: return self._add_operation_to_queue(operation) def get_queue_size(self) -> int: + """ + Returns the current size of the operation queue. + """ # CRITICAL SECTION self._operation_queue_lock.acquire() size : int = len(self._operation_queue) @@ -428,4 +752,7 @@ def get_queue_size(self) -> int: return size def get_queue_limit(self) -> int: + """ + Returns the maximum size of this operation queue. + """ return self._queue_limit_size diff --git a/samples/operation_queue/operation_queue.py b/samples/operation_queue/operation_queue.py index a3bfcf4c..7ce2b974 100644 --- a/samples/operation_queue/operation_queue.py +++ b/samples/operation_queue/operation_queue.py @@ -78,7 +78,7 @@ def on_queue_empty(): if __name__ == '__main__': mqtt_connection = cmdUtils.build_mqtt_connection(on_connection_interrupted, on_connection_resumed) - queue_builder = mqtt_operation_queue.OperationQueueBuilder() + queue_builder = mqtt_operation_queue.MqttOperationQueueBuilder() queue_builder.with_connection(mqtt_connection).with_queue_limit_size(cmdUtils.get_command("queue_limit")) queue_builder.with_on_queue_empty_callback(on_queue_empty) if (cmdUtils.get_command("queue_mode") == 0): @@ -93,7 +93,7 @@ def on_queue_empty(): elif (cmdUtils.get_command("queue_mode") == 3): queue_builder.with_queue_insert_behavior(mqtt_operation_queue.InsertBehavior.INSERT_FRONT) queue_builder.with_queue_limit_behavior(mqtt_operation_queue.LimitBehavior.DROP_BACK) - mqtt_queue = queue_builder.build() + mqtt_queue : mqtt_operation_queue.MqttOperationQueue = queue_builder.build() if is_ci == False: print("Connecting to {} with client ID '{}'...".format( From 05d9d63ca2ff3cead09c16ad9b7f53042e362d7f Mon Sep 17 00:00:00 2001 From: Noah Beard Date: Thu, 9 Mar 2023 14:42:15 -0500 Subject: [PATCH 03/12] Add operation queue README --- samples/operation_queue/README.md | 136 ++++++++++++++++++++++++++++++ 1 file changed, 136 insertions(+) create mode 100644 samples/operation_queue/README.md diff --git a/samples/operation_queue/README.md b/samples/operation_queue/README.md new file mode 100644 index 00000000..4ebc8453 --- /dev/null +++ b/samples/operation_queue/README.md @@ -0,0 +1,136 @@ +# Operations Queue + +[**Return to main sample list**](../README.md) + +This sample uses the +[Message Broker](https://docs.aws.amazon.com/iot/latest/developerguide/iot-message-broker.html) +for AWS IoT to send and receive messages through an MQTT connection. It then subscribes and begins publishing messages to a topic, like in the [PubSub sample](../PubSub/README.md). + +However, this sample uses a operation queue to handle the processing of operations, rather than directly using the MQTT311 connection. This gives an extreme level of control over how operations are processed, the order they are processed in, limiting how many operations can exist waiting to be sent, what happens when a new operation is added when the queue is full, and ensuring the MQTT311 connection is never overwhelmed with too many messages at once. + +Additionally, using a queue allows you to put limits on how much data you are trying to send through the socket to the AWS IoT Core server. This can help keep your application within the IoT Core sending limits, ensuring all your MQTT311 operations are being processed correctly and the socket is not backed up. It also the peace of mind that your application cannot become "runaway" and start sending an extreme amount of messages at all once, clogging the socket depending on how large the messages are and the frequency. + +**Note**: MQTT5 does not have the same issues with backed up sockets due to the use of an internal operation queue, which ensures the socket does not get backed up. + +This operation queue can be configured in a number of different ways to best match the needs of your application. Further, the operation queue is designed to be as standalone as possible so it can be used as a starting point for implementing your own operation queue for the MQTT311 connection. The `MqttOperationQueue` class is fully documented with comments explaining the functions used. + +Your IoT Core Thing's [Policy](https://docs.aws.amazon.com/iot/latest/developerguide/iot-policies.html) must provide privileges for this sample to connect, subscribe, publish, and receive. Below is a sample policy that can be used on your IoT Core Thing that will allow this sample to run as intended. + +
+(see sample policy) +
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Action": [
+        "iot:Publish",
+        "iot:Receive"
+      ],
+      "Resource": [
+        "arn:aws:iot:region:account:topic/test/topic/*"
+      ]
+    },
+    {
+      "Effect": "Allow",
+      "Action": [
+        "iot:Subscribe"
+      ],
+      "Resource": [
+        "arn:aws:iot:region:account:topicfilter/test/topic/*"
+      ]
+    },
+    {
+      "Effect": "Allow",
+      "Action": [
+        "iot:Connect"
+      ],
+      "Resource": [
+        "arn:aws:iot:region:account:client/test-*"
+      ]
+    }
+  ]
+}
+
+ +Replace with the following with the data from your AWS account: +* ``: The AWS IoT Core region where you created your AWS IoT Core thing you wish to use with this sample. For example `us-east-1`. +* ``: Your AWS IoT Core account ID. This is the set of numbers in the top right next to your AWS account name when using the AWS IoT Core website. + +Note that in a real application, you may want to avoid the use of wildcards in your ClientID or use them selectively. Please follow best practices when working with AWS on production applications using the SDK. Also, for the purposes of this sample, please make sure your policy allows a client ID of `test-*` to connect or use `--client_id ` to send the client ID your policy supports. + +
+ +## How to run + +To Run this sample, use the following command: + +```sh +python3 samples/operation_queue/operation_queue.py --endpoint --cert --key +``` + +You can also pass a Certificate Authority file (CA) if your certificate and key combination requires it: + +```sh +python3 samples/operation_queue/operation_queue.py --endpoint --cert --key --ca_file +``` + +Finally, you can control how the operation queue inserts new operations and drops operations when the queue is full via the `--queue_mode` parameter. For example, to have a rolling queue where new operations are added to the front and overflow is removed from the back of the queue: + +```sh +python3 samples/operation_queue/operation_queue.py --endpoint --cert --key --queue_mode 1 +``` + +See the output of the `--help` argument for more information on the queue operation modes and configuration of this sample. + + +## Queue Design + +The operation queue is designed to hold a number of operations (publish, subscribe, and unsubscribe) in a queue so that it can be processed in a controlled manner that doesn't overwhelm the MQTT311 connection, as well as giving a control to your code on how the operations are processed. This is written on top of the MQTT311 connection and as a sample so it can be used as a reference and extended/adjusted to meet the needs of your application. + +The backbone of how the operation queue works is the [MQTT311 operation statistics](https://awslabs.github.io/aws-crt-python/api/mqtt.html#awscrt.mqtt.Connection.get_stats). [These statistics](https://awslabs.github.io/aws-crt-python/api/mqtt.html#awscrt.mqtt.OperationStatisticsData) reported by the MQTT311 connection give a window into what the MQTT311 connection is doing, what operations are being sent to AWS IoT Core via via a socket, and what operations are waiting for responses from AWS IoT Core. This is how the operation queue knows how many operations are being processed and when to send another, as well as allowing it to be reactive to the state of the connection. + +Specifically, the operation statistics is how the operation queue class calculates whether or not to send another operation in its queue to the MQTT311 connection or not. If the MQTT311 connection has many operations waiting, it can hold off and wait until the MQTT311 connection has processed the data and can consume more. This has the benefits mentioned in the top of this document: It prevents the MQTT311 connection from being flooded with too many operations and the socket being too backed up. Additionally, it has the benefit of allowing your code to control the order of operations that are sent directly and can be configured so that your code has the assurance it will not send too much data and exceed AWS IoT Core limits. + +Put simply, the operation queue is a system that looks at the MQTT311 operation statistics at a timed interval and determines whether the MQTT311 connection can take another operation based on the settings of the operation queue. If it can, it sends that operation to the MQTT311 client and continues to observe it's statistics until either the queue is empty and there is nothing to do, or until the operation queue is stopped. + +___________ + +The operation queue is designed to be a helpful reference you can use directly in your applications to have a queue-based control over the MQTT operation flow, as well as ensuring that you have back pressure support and will not write too much data to the socket at a given time. The operation queue in this sample is designed to be flexible and powerful, but of course you can extend the operation queue to meet the needs of your application. For example, it could be extended to allow injecting operations at specific indexes in the queue, allow removing operations at any index in the queue, etc. All of the code for the operation queue has comments that explain what each function does for easier customization and extension. + +Below is more information on specifics about how the operation queue works. + +### Operations outside of the queue + +What is great with using the operation statistics to determine the state of the MQTT311 connection is that if your code uses the MQTT311 connection directly for something or you are doing non-queued operations, like connect/disconnect for example, then the queue will still properly react and possibly limit the action of the queue based on what the statistics return and how your queue is setup. You can even have two operation queues on the same connection! + +This is helpful because it allows you to selectively choose which operations are written to the MQTT311 connection socket right away and which are behind the queue on a per-operation basis. This means that if you have a operation that you must get sent as soon as possible regardless of any queues, you can just directly call the operation (publish, subscribe, unsubscribe, etc.) directly on the MQTT311 connection and the queue will react accordingly. + +### Retried operations + +A question you might be wondering is how does the operation queue, and by extension the operation statistics, handle retried QoS1 operations? QoS1 requires getting an acknowledgement (ACK) back from the MQTT server and, should this not happen, it will retry the operation by sending it again. This is why QoS1 is described as "at least once", because it only tries to guarantee that the message will be sent at **least once** but there are no guarantees that it will not be sent **more than once**. + +**Note**: For operations that need to be sent exactly and only one time, QoS2 would be used for this purpose. However, at this time QoS2 is not currently supported by the AWS IoT SDKs. + +For the MQTT311 connection operations (publish, subscribe, and unsubscribe), QoS1 operations will NOT be retried automatically even if the MQTT server does not send an acknowledgement. These operations will keep waiting for a reply from the server until it receives one. This can be adjusted by setting an operation timeout, in which case the operation will be resolved as having failed to send if an acknowledgment is not received in the given timeout time. Note that, by default, the MQTT311 connection will not have an operation timeout set and this timeout has to be manually set. + +Connections are an exception to this however. Connections are retried automatically with an exceptional back-off if the connect does not get an acknowledgment from the server in the expected time. However, because this process is done internally by the MQTT311 connection, it would fall into the [operations outside of the queue](#operations-outside-of-the-queue) for the purposes of the operation queue. + +However, there is a case where QoS1 operations (publish, subscribe, unsubscribe) will be retried, and this is when the MQTT311 connection has in-flight operations that have not received an acknowledgment from the server and the MQTT311 connection becomes disconnected from the server. When the MQTT311 connection reconnects, it will resend the QoS1 operations that were waiting for acknowledgements prior to becoming disconnected. + +When a QoS1 operation is made and sent to the socket so it can go to the MQTT server, it gets added to the "in-flight" operation statistics and is removed from the "incomplete operations". If the server does not send an acknowledgement of the operation, the MQTT311 client becomes disconnected, the MQTT311 reconnects, and the operation is resent, the operation is NOT moved from the "in-flight" statistics nor is a new one added. Instead, the resent operation simply keeps waiting in the "in-flight". Finally, when the operation gets a response from the server, whether it be on the initial operation or a retry, it will be removed from the "in-flight" statistics. Likewise, if the operation has a timeout set and does not get an acknowledgement within the given timeout period, it will be removed from the "in-flight" statistics when the operation is considered to have failed. + +**Note**: This only applies to QoS1 operations! QoS0 operations are different and [explained below](#qos-0-operations). + +This means that for the operation queue, it has no idea if the operation has been retried or not, it just sees it as there being an in-flight operation. This means that if you have a bunch of in-flight operations all waiting, the queue will wait for the MQTT server to send acknowledgements without needing additional configurations nor code on the user side. + +### QoS 0 operations + +Operations can be made with either QoS0 or QoS1. QoS0 states than an operation will be sent "at most once". This means that once the operation is written to the socket, it is removed and the code does not wait to see if the server actually got the data. For the MQTT311 operation statistics, it means that a QoS0 operation is written to the socket and then immediately removed from the "incomplete operations", it does not get added to "in-flight" nor is there any waiting to be done, it is fire and forget. + +For the operation queue, this means that QoS0 operations are stored in the queue and will wait like all other operations until the MQTT311 operation statistics are in a state where it can be published, but once it is published, it will be fired and forgotten. QoS0 operations will not hang around and will be fired as soon as they are in the front of the queue and the MQTT311 operation statistics are in an acceptable state. + +### Service Clients (Shadow, Jobs, etc) + +The operation queue should work fully alongside service clients like [Shadow](../Shadow/README.md), [Jobs](../Jobs/README.md), and [Fleet Provisioning](../Identity/README.md). These service clients ultimately subscribe, publish, and unsubscribe to MQTT topics, and as such they are compatible with the operation queue. That said though, they will **not** be added to the operation queue, instead they will function like if operations were made manually outside of the queue, as noted in the [Operations outside of the queue section](#operations-outside-of-the-queue). From 7bd18e0de32735012f2401c573e12338b4058a41 Mon Sep 17 00:00:00 2001 From: Noah Beard Date: Thu, 9 Mar 2023 15:47:58 -0500 Subject: [PATCH 04/12] Fix overflow return bug, add sample tests --- .../operation_queue/mqtt_operation_queue.py | 8 +- .../mqtt_operation_queue_tests.py | 271 ++++++++++++++++++ samples/operation_queue/operation_queue.py | 7 +- 3 files changed, 282 insertions(+), 4 deletions(-) diff --git a/samples/operation_queue/mqtt_operation_queue.py b/samples/operation_queue/mqtt_operation_queue.py index 64a45333..86f4aa00 100644 --- a/samples/operation_queue/mqtt_operation_queue.py +++ b/samples/operation_queue/mqtt_operation_queue.py @@ -415,6 +415,8 @@ def _add_operation_to_queue_overflow(self, operation: QueueOperation) -> tuple: Used to make separate the logic for when the queue is full from when it is not yet full. Called by _add_operation_to_queue. + Returns tuple with [result, dropped operation (or None)] + Keyword Args: operation (QueueOperation): The operation to add. """ @@ -426,7 +428,7 @@ def _add_operation_to_queue_overflow(self, operation: QueueOperation) -> tuple: result[1] = self._operation_queue[0] del self._operation_queue[0] self._print_log_message(f"Dropped operation of type {result[1].type} from the front...") - result[0] = self._add_operation_to_queue_insert(operation) + result[0] = self._add_operation_to_queue_insert(operation) elif (self._queue_limit_behavior == LimitBehavior.DROP_BACK): end_of_queue = len(self._operation_queue)-1 result[1] = self._operation_queue[end_of_queue] @@ -461,8 +463,8 @@ def _add_operation_to_queue(self, operation: QueueOperation) -> QueueResult: result = self._add_operation_to_queue_insert(operation) else: return_data = self._add_operation_to_queue_overflow(operation) - dropped_operation = return_data[0] - result = return_data[1] + dropped_operation = return_data[1] + result = return_data[0] except Exception as exception: self._print_log_message(f"Exception ocurred adding operation to queue. Exception: {exception}") diff --git a/samples/operation_queue/mqtt_operation_queue_tests.py b/samples/operation_queue/mqtt_operation_queue_tests.py index e69de29b..b63fe089 100644 --- a/samples/operation_queue/mqtt_operation_queue_tests.py +++ b/samples/operation_queue/mqtt_operation_queue_tests.py @@ -0,0 +1,271 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0. + +from awscrt import mqtt +from uuid import uuid4 +from concurrent.futures import Future +import os + +import mqtt_operation_queue +from command_line_utils import CommandLineUtils + +TEST_TOPIC = "test/topic/" + str(uuid4()) +PRINT_QUEUE_LOGS = False + +class tester: + connection: mqtt.Connection + on_queue_empty_future : Future = Future() + on_queue_sent_future : Future = Future() + on_queue_dropped_future : Future = Future() + cmdUtils : CommandLineUtils + + def on_application_failure(self, test_name : str, error : Exception): + print (f"Error in test {test_name}: {error}") + os._exit(-1) + + def test_connection_setup(self): + self.connection = self.cmdUtils.build_mqtt_connection(None, None) + self.on_queue_empty_future : Future = Future() + self.on_queue_sent_future : Future = Future() + self.on_queue_dropped_future : Future = Future() + + def test_operation_success(self, result : mqtt_operation_queue.QueueResult, test_name: str): + if (result != mqtt_operation_queue.QueueResult.SUCCESS): + self.on_application_failure(test_name, ValueError(f"operation was not successful. Result: {result}", test_name)) + + def test_connect_sub_pub_unsub(self): + self.test_connection_setup() + try: + def on_queue_empty_future(): + self.on_queue_empty_future.set_result(None) + def on_queue_sent_future(operation: mqtt_operation_queue.QueueOperation, _): + self.on_queue_sent_future.set_result(operation) + def on_queue_dropped_future(operation: mqtt_operation_queue.QueueOperation): + self.on_queue_dropped_future.set_result(operation) + + queue_builder = mqtt_operation_queue.MqttOperationQueueBuilder() + queue_builder.with_connection(self.connection).with_enable_logging(PRINT_QUEUE_LOGS) + queue_builder.with_on_queue_empty_callback(on_queue_empty_future) + queue_builder.with_on_operation_sent_callback(on_queue_sent_future) + queue_builder.with_on_operation_dropped_callback(on_queue_dropped_future) + mqtt_queue : mqtt_operation_queue.MqttOperationQueue = queue_builder.build() + + # Add the operations to the queue + self.test_operation_success(mqtt_queue.subscribe(TEST_TOPIC, mqtt.QoS.AT_LEAST_ONCE, None), "test_connect_sub_pub_unsub") + self.test_operation_success(mqtt_queue.publish(TEST_TOPIC, "hello_world", mqtt.QoS.AT_LEAST_ONCE), "test_connect_sub_pub_unsub") + self.test_operation_success(mqtt_queue.unsubscribe(TEST_TOPIC), "test_connect_sub_pub_unsub") + + if (mqtt_queue.get_queue_size() != 3): + self.on_application_failure("test_connect_sub_pub_unsub", ValueError("Queue size is not 3")) + + connect_future = self.connection.connect() + connect_future.result() + mqtt_queue.start() + + # Make sure the order is right. Order should be: Sub, Pub, Unsub + return_operation : mqtt_operation_queue.QueueOperation = self.on_queue_sent_future.result() + if (return_operation == None or return_operation.type != mqtt_operation_queue.QueueOperationType.SUBSCRIBE): + self.on_application_failure("test_connect_sub_pub_unsub", ValueError(f"First operation is not subscribe. Type is {return_operation.type}")) + self.on_queue_sent_future = Future() # reset future + return_operation = self.on_queue_sent_future.result() + if (return_operation == None or return_operation.type != mqtt_operation_queue.QueueOperationType.PUBLISH): + self.on_application_failure("test_connect_sub_pub_unsub", ValueError(f"Second operation is not publish. Type is {return_operation.type}")) + self.on_queue_sent_future = Future() # reset future + return_operation = self.on_queue_sent_future.result() + if (return_operation == None or return_operation.type != mqtt_operation_queue.QueueOperationType.UNSUBSCRIBE): + self.on_application_failure("test_connect_sub_pub_unsub", ValueError(f"Third operation is not unsubscribe. Type is {return_operation.type}")) + + self.on_queue_empty_future.result() + mqtt_queue.stop() + disconnect_future = self.connection.disconnect() + disconnect_future.result() + except Exception as ex: + self.on_application_failure("test_connect_sub_pub_unsub", ex) + + def test_drop_back(self): + self.test_connection_setup() + try: + def on_queue_empty_future(): + self.on_queue_empty_future.set_result(None) + def on_queue_sent_future(operation: mqtt_operation_queue.QueueOperation, _): + self.on_queue_sent_future.set_result(operation) + def on_queue_dropped_future(operation: mqtt_operation_queue.QueueOperation): + self.on_queue_dropped_future.set_result(operation) + + queue_builder = mqtt_operation_queue.MqttOperationQueueBuilder() + queue_builder.with_connection(self.connection).with_enable_logging(PRINT_QUEUE_LOGS) + queue_builder.with_on_queue_empty_callback(on_queue_empty_future) + queue_builder.with_on_operation_sent_callback(on_queue_sent_future) + queue_builder.with_on_operation_dropped_callback(on_queue_dropped_future) + queue_builder.with_queue_limit_size(2).with_queue_limit_behavior(mqtt_operation_queue.LimitBehavior.DROP_BACK) + mqtt_queue : mqtt_operation_queue.MqttOperationQueue = queue_builder.build() + + # Add the operations to the queue + self.test_operation_success(mqtt_queue.subscribe(TEST_TOPIC, mqtt.QoS.AT_LEAST_ONCE, None), "test_drop_back") + + # Add 10 publishes + for i in range(0, 10): + self.on_queue_dropped_future = Future() # reset future + self.test_operation_success(mqtt_queue.publish(TEST_TOPIC, "hello_world", mqtt.QoS.AT_LEAST_ONCE), "test_drop_back") + + if (mqtt_queue.get_queue_size() != 2): + self.on_application_failure("test_drop_back", ValueError("Queue size is not 2")) + + self.on_queue_dropped_future = Future() # reset future + self.test_operation_success(mqtt_queue.unsubscribe(TEST_TOPIC), "test_drop_back") + dropped_operation : mqtt_operation_queue.QueueOperation = self.on_queue_dropped_future.result() + if (dropped_operation == None or dropped_operation.type != mqtt_operation_queue.QueueOperationType.PUBLISH): + self.on_application_failure("test_drop_back", ValueError(f"Dropped operation is not publish. Type is {dropped_operation.type}")) + + connect_future = self.connection.connect() + connect_future.result() + mqtt_queue.start() + + # Make sure the order is right. Order should be: Sub, Unsub + return_operation : mqtt_operation_queue.QueueOperation = self.on_queue_sent_future.result() + if (return_operation == None or return_operation.type != mqtt_operation_queue.QueueOperationType.SUBSCRIBE): + self.on_application_failure("test_drop_back", ValueError(f"First operation is not subscribe. Type is {return_operation.type}")) + self.on_queue_sent_future = Future() # reset future + return_operation = self.on_queue_sent_future.result() + if (return_operation == None or return_operation.type != mqtt_operation_queue.QueueOperationType.UNSUBSCRIBE): + self.on_application_failure("test_drop_back", ValueError(f"Second operation is not unsubscribe. Type is {return_operation.type}")) + + self.on_queue_empty_future.result() + mqtt_queue.stop() + disconnect_future = self.connection.disconnect() + disconnect_future.result() + except Exception as ex: + self.on_application_failure("test_drop_back", ex) + + def test_drop_front(self): + self.test_connection_setup() + try: + def on_queue_dropped_future(operation: mqtt_operation_queue.QueueOperation): + self.on_queue_dropped_future.set_result(operation) + + queue_builder = mqtt_operation_queue.MqttOperationQueueBuilder() + queue_builder.with_connection(self.connection).with_enable_logging(PRINT_QUEUE_LOGS) + queue_builder.with_on_operation_dropped_callback(on_queue_dropped_future) + queue_builder.with_queue_limit_size(2).with_queue_limit_behavior(mqtt_operation_queue.LimitBehavior.DROP_FRONT) + mqtt_queue : mqtt_operation_queue.MqttOperationQueue = queue_builder.build() + + # Add the operations to the queue + self.test_operation_success(mqtt_queue.subscribe(TEST_TOPIC, mqtt.QoS.AT_LEAST_ONCE, None), "test_drop_front") + self.test_operation_success(mqtt_queue.unsubscribe(TEST_TOPIC), "test_drop_front") + + if (mqtt_queue.get_queue_size() != 2): + self.on_application_failure("test_drop_front", ValueError("Queue size is not 2")) + + # Add two publishes, make sure drop order is correct + self.test_operation_success(mqtt_queue.publish(TEST_TOPIC, "hello_world", mqtt.QoS.AT_LEAST_ONCE), "test_drop_front") + dropped_operation : mqtt_operation_queue.QueueOperation = self.on_queue_dropped_future.result() + if (dropped_operation == None or dropped_operation.type != mqtt_operation_queue.QueueOperationType.SUBSCRIBE): + self.on_application_failure("test_drop_front", ValueError(f"First Dropped operation is not subscribe. Type is {dropped_operation.type}")) + # second drop + self.on_queue_dropped_future = Future() # reset future + self.test_operation_success(mqtt_queue.publish(TEST_TOPIC, "hello_world", mqtt.QoS.AT_LEAST_ONCE), "test_drop_front") + dropped_operation = self.on_queue_dropped_future.result() + if (dropped_operation == None or dropped_operation.type != mqtt_operation_queue.QueueOperationType.UNSUBSCRIBE): + self.on_application_failure("test_drop_front", ValueError(f"First Dropped operation is not unsubscribe. Type is {dropped_operation.type}")) + + except Exception as ex: + self.on_application_failure("test_drop_front", ex) + + def test_add_front(self): + self.test_connection_setup() + try: + def on_queue_empty_future(): + self.on_queue_empty_future.set_result(None) + def on_queue_sent_future(operation: mqtt_operation_queue.QueueOperation, _): + self.on_queue_sent_future.set_result(operation) + + queue_builder = mqtt_operation_queue.MqttOperationQueueBuilder() + queue_builder.with_connection(self.connection).with_enable_logging(PRINT_QUEUE_LOGS) + queue_builder.with_on_queue_empty_callback(on_queue_empty_future) + queue_builder.with_on_operation_sent_callback(on_queue_sent_future) + queue_builder.with_queue_limit_size(2).with_queue_limit_behavior(mqtt_operation_queue.LimitBehavior.DROP_BACK) + queue_builder.with_queue_insert_behavior(mqtt_operation_queue.InsertBehavior.INSERT_FRONT) + mqtt_queue : mqtt_operation_queue.MqttOperationQueue = queue_builder.build() + + # Fill with publishes + self.test_operation_success(mqtt_queue.publish(TEST_TOPIC, "hello_world", mqtt.QoS.AT_LEAST_ONCE), "test_add_front") + self.test_operation_success(mqtt_queue.publish(TEST_TOPIC, "hello_world", mqtt.QoS.AT_LEAST_ONCE), "test_add_front") + + if (mqtt_queue.get_queue_size() != 2): + self.on_application_failure("test_add_front", ValueError("Queue size is not 2")) + + # Add unsubscribe than subscribe, which should result in the queue order of subscribe, unsubscribe + self.test_operation_success(mqtt_queue.unsubscribe(TEST_TOPIC), "test_add_front") + self.test_operation_success(mqtt_queue.subscribe(TEST_TOPIC, mqtt.QoS.AT_LEAST_ONCE, None), "test_add_front") + + if (mqtt_queue.get_queue_size() != 2): + self.on_application_failure("test_add_front", ValueError("Queue size is not 2")) + + connect_future = self.connection.connect() + connect_future.result() + mqtt_queue.start() + + # Make sure the order is right. Order should be: Sub, Unsub + return_operation : mqtt_operation_queue.QueueOperation = self.on_queue_sent_future.result() + if (return_operation == None or return_operation.type != mqtt_operation_queue.QueueOperationType.SUBSCRIBE): + self.on_application_failure("test_add_front", ValueError(f"First operation is not subscribe. Type is {return_operation.type}")) + self.on_queue_sent_future = Future() # reset future + return_operation = self.on_queue_sent_future.result() + if (return_operation == None or return_operation.type != mqtt_operation_queue.QueueOperationType.UNSUBSCRIBE): + self.on_application_failure("test_add_front", ValueError(f"Second operation is not unsubscribe. Type is {return_operation.type}")) + + self.on_queue_empty_future.result() + mqtt_queue.stop() + + disconnect_future = self.connection.disconnect() + disconnect_future.result() + + except Exception as ex: + self.on_application_failure("test_add_front", ex) + + def test_add_error(self): + self.test_connection_setup() + try: + queue_builder = mqtt_operation_queue.MqttOperationQueueBuilder() + queue_builder.with_connection(self.connection).with_enable_logging(PRINT_QUEUE_LOGS) + queue_builder.with_queue_limit_size(2).with_queue_limit_behavior(mqtt_operation_queue.LimitBehavior.RETURN_ERROR) + mqtt_queue : mqtt_operation_queue.MqttOperationQueue = queue_builder.build() + # Fill with unsubscribe + self.test_operation_success(mqtt_queue.unsubscribe(TEST_TOPIC), "test_add_error") + self.test_operation_success(mqtt_queue.unsubscribe(TEST_TOPIC), "test_add_error") + # Try to add another but it should return error stating queue is full + operation_result : mqtt_operation_queue.QueueResult = mqtt_queue.unsubscribe(TEST_TOPIC) + if (operation_result != mqtt_operation_queue.QueueResult.ERROR_QUEUE_FULL): + self.on_application_failure("test_add_error", ValueError("Did not return queue full error trying to add operation to full queue")) + + except Exception as ex: + self.on_application_failure("test_add_error", ex) + + def perform_tests(self): + print("Starting test_connect_sub_pub_unsub test") + self.test_connect_sub_pub_unsub() + print ("Finished test_connect_sub_pub_unsub test") + + print("Starting test_drop_back test") + self.test_drop_back() + print ("Finished test_drop_back test") + + print("Starting test_drop_front test") + self.test_drop_front() + print ("Finished test_drop_front test") + + print("Starting test_add_front test") + self.test_add_front() + print ("Finished test_add_front test") + + print("Starting test_add_error test") + self.test_add_error() + print ("Finished test_add_error test") + + +def perform_tests(cmdUtils : CommandLineUtils): + tests = tester() + tests.cmdUtils = cmdUtils + tests.perform_tests() + print ("All tests finished. Exiting...") + os._exit(0) diff --git a/samples/operation_queue/operation_queue.py b/samples/operation_queue/operation_queue.py index 7ce2b974..f41183fd 100644 --- a/samples/operation_queue/operation_queue.py +++ b/samples/operation_queue/operation_queue.py @@ -14,6 +14,7 @@ # since it is subscribed to that same topic. import mqtt_operation_queue +import mqtt_operation_queue_tests # Parse arguments from command_line_utils import CommandLineUtils @@ -35,7 +36,8 @@ "\n\t3 = Overflow removes from queue back and new operations are pushed to queue front", default=10, type=int) cmdUtils.register_command("run_tests", "", - "If set to True (1 or greater), then the queue tests will be run instead of the sample (optional, default=0)") + "If set to True (1 or greater), then the queue tests will be run instead of the sample (optional, default=0)", + default=0, type=int) cmdUtils.register_command("is_ci", "", "If present the sample will run in CI mode (optional, default='None')") # Needs to be called so the command utils parse the commands cmdUtils.get_args() @@ -44,6 +46,9 @@ received_all_event = threading.Event() is_ci = cmdUtils.get_command("is_ci", None) != None +if (cmdUtils.get_command("run_tests") > 0): + mqtt_operation_queue_tests.perform_tests(cmdUtils) + # Callback when connection is accidentally lost. def on_connection_interrupted(connection, error, **kwargs): print("Connection interrupted. error: {}".format(error)) From d27031cd64a0ed8ce4ae4d2a1087ed33b42ddec0 Mon Sep 17 00:00:00 2001 From: Noah Beard Date: Thu, 9 Mar 2023 15:56:52 -0500 Subject: [PATCH 05/12] Format files, add all callbacks to sample --- .../operation_queue/mqtt_operation_queue.py | 196 ++++++++-------- .../mqtt_operation_queue_tests.py | 219 ++++++++++++------ samples/operation_queue/operation_queue.py | 53 +++-- 3 files changed, 289 insertions(+), 179 deletions(-) diff --git a/samples/operation_queue/mqtt_operation_queue.py b/samples/operation_queue/mqtt_operation_queue.py index 86f4aa00..78faf93a 100644 --- a/samples/operation_queue/mqtt_operation_queue.py +++ b/samples/operation_queue/mqtt_operation_queue.py @@ -12,45 +12,50 @@ # Enums ######################################################## + class QueueResult(Enum): """ The result of attempting to perform an operation on the MqttOperationQueue. The value indicates either success or what type of issue was encountered. """ - SUCCESS=0 - ERROR_QUEUE_FULL=1 - ERROR_INVALID_ARGUMENT=2 - UNKNOWN_QUEUE_LIMIT_BEHAVIOR=3 - UNKNOWN_QUEUE_INSERT_BEHAVIOR=4 - UNKNOWN_OPERATION=5 - UNKNOWN_ERROR=6 + SUCCESS = 0 + ERROR_QUEUE_FULL = 1 + ERROR_INVALID_ARGUMENT = 2 + UNKNOWN_QUEUE_LIMIT_BEHAVIOR = 3 + UNKNOWN_QUEUE_INSERT_BEHAVIOR = 4 + UNKNOWN_OPERATION = 5 + UNKNOWN_ERROR = 6 + class QueueOperationType(Enum): """ An enum to indicate the type of data the QueueOperation contains. Used to differentiate between different operations in a common blob object. """ - NONE=0 - PUBLISH=1 - SUBSCRIBE=2 - UNSUBSCRIBE=3 + NONE = 0 + PUBLISH = 1 + SUBSCRIBE = 2 + UNSUBSCRIBE = 3 + class LimitBehavior(Enum): """ An enum to indicate what happens when the MqttOperationQueue is completely full but new operations are requested to be added to the queue. """ - DROP_FRONT=0 - DROP_BACK=1 - RETURN_ERROR=2 + DROP_FRONT = 0 + DROP_BACK = 1 + RETURN_ERROR = 2 + class InsertBehavior(Enum): """ An enum to indicate what happens when the MqttOperationQueue has a new operation it needs to add to the queue, configuring where the new operation is added. """ - INSERT_FRONT=0 - INSERT_BACK=1 + INSERT_FRONT = 0 + INSERT_BACK = 1 + class QueueOperation(): """ @@ -58,17 +63,18 @@ class QueueOperation(): an enum to indicate what type of operation should be stored within. Used to provide a common base that all operations can be derived from. """ - type : QueueOperationType = QueueOperationType.NONE - topic : str = "" + type: QueueOperationType = QueueOperationType.NONE + topic: str = "" payload: any = None - qos : mqtt.QoS = mqtt.QoS.AT_MOST_ONCE - retain : bool = None - subscribe_callback : FunctionType = None + qos: mqtt.QoS = mqtt.QoS.AT_MOST_ONCE + retain: bool = None + subscribe_callback: FunctionType = None ####################################################### # Classes ######################################################## + class MqttOperationQueueBuilder: """ A builder that contains all of the options of the MqttOperationQueue. @@ -76,21 +82,21 @@ class MqttOperationQueueBuilder: MqttOperationQueue with the build() function. """ - _connection : mqtt.Connection = None - _queue_limit_size : int = 10 - _queue_limit_behavior : LimitBehavior = LimitBehavior.DROP_BACK - _queue_insert_behavior : InsertBehavior = InsertBehavior.INSERT_BACK - _incomplete_limit : int = 1 - _inflight_limit : int = 1 - _on_operation_sent_callback : FunctionType = None - _on_operation_sent_failure_callback : FunctionType = None - _on_operation_dropped_callback : FunctionType = None - _on_queue_full_callback : FunctionType = None - _on_queue_empty_callback : FunctionType = None - _queue_loop_time_ms : int = 1000 - _enable_logging : bool = False - - def with_connection(self, connection:mqtt.Connection): + _connection: mqtt.Connection = None + _queue_limit_size: int = 10 + _queue_limit_behavior: LimitBehavior = LimitBehavior.DROP_BACK + _queue_insert_behavior: InsertBehavior = InsertBehavior.INSERT_BACK + _incomplete_limit: int = 1 + _inflight_limit: int = 1 + _on_operation_sent_callback: FunctionType = None + _on_operation_sent_failure_callback: FunctionType = None + _on_operation_dropped_callback: FunctionType = None + _on_queue_full_callback: FunctionType = None + _on_queue_empty_callback: FunctionType = None + _queue_loop_time_ms: int = 1000 + _enable_logging: bool = False + + def with_connection(self, connection: mqtt.Connection): """ Sets the mqtt.Connection that will be used by the MqttOperationQueue. This is a REQUIRED argument that has to be set in order for the MqttOperationQueue to function. @@ -107,7 +113,7 @@ def get_connection(self) -> mqtt.Connection: """ return self._connection - def with_queue_limit_size(self, queue_limit_size:int): + def with_queue_limit_size(self, queue_limit_size: int): """ Sets the maximum size of the operation queue in the MqttOperationQueue. Default operation queue size is 10. @@ -127,7 +133,7 @@ def get_connection(self) -> int: """ return self._queue_limit_size - def with_queue_limit_behavior(self, queue_limit_behavior:LimitBehavior): + def with_queue_limit_behavior(self, queue_limit_behavior: LimitBehavior): """ Sets how the MqttOperationQueue will behave when the operation queue is full but a new operation is requested to be added to the queue. @@ -145,7 +151,7 @@ def get_queue_limit_behavior(self) -> LimitBehavior: """ return self._queue_limit_behavior - def with_queue_insert_behavior(self, queue_insert_behavior:InsertBehavior): + def with_queue_insert_behavior(self, queue_insert_behavior: InsertBehavior): """ Sets how the MqttOperationQueue will behave when inserting a new operation into the queue. The default is INSERT_BACK, which will add the new operation to the back (last to be executed) of the queue. @@ -162,7 +168,7 @@ def get_queue_insert_behavior(self) -> InsertBehavior: """ return self._queue_insert_behavior - def with_incomplete_limit(self, incomplete_limit:int): + def with_incomplete_limit(self, incomplete_limit: int): """ Sets the maximum number of incomplete operations that the MQTT connection can have before the MqttOperationQueue will wait for them to be complete. Incomplete operations are those that have been @@ -185,7 +191,7 @@ def get_incomplete_limit(self) -> int: """ return self._incomplete_limit - def with_inflight_limit(self, inflight_limit:int): + def with_inflight_limit(self, inflight_limit: int): """ Sets the maximum number of inflight operations that the MQTT connection can have before the MqttOperationQueue will wait for them to be complete. inflight operations are those that have been @@ -272,7 +278,7 @@ def get_on_operation_dropped_callback(self) -> FunctionType: """ return self._on_operation_dropped_callback - def with_on_queue_full_callback(self, on_queue_full_callback:FunctionType): + def with_on_queue_full_callback(self, on_queue_full_callback: FunctionType): """ Sets the callback that will be invoked when the operation queue is full. @@ -291,7 +297,7 @@ def get_on_queue_full_callback(self) -> FunctionType: """ return self._on_queue_full_callback - def with_on_queue_empty_callback(self, on_queue_empty_callback:FunctionType): + def with_on_queue_empty_callback(self, on_queue_empty_callback: FunctionType): """ Sets the callback that will be invoked when the operation queue is completely empty. @@ -310,7 +316,7 @@ def get_on_queue_empty_callback(self) -> FunctionType: """ return self._on_queue_empty_callback - def with_queue_loop_time(self, queue_loop_time_ms:int): + def with_queue_loop_time(self, queue_loop_time_ms: int): """ Sets the interval, in milliseconds, that the MqttOperationQueue will wait before checking the queue and (possibly) processing an operation based on the statistics and state of the MqttClientConnection assigned to the MqttOperationQueue. @@ -328,7 +334,7 @@ def get_queue_loop_time(self) -> int: """ return self._queue_loop_time_ms - def with_enable_logging(self, enable_logging:bool): + def with_enable_logging(self, enable_logging: bool): """ Sets whether the MqttOperationQueue will print logging statements to help debug and determine how the MqttOperationQueue is functioning. @@ -353,26 +359,26 @@ def build(self): class MqttOperationQueue: - _operation_queue : list[QueueOperation] = [] - _operation_queue_lock : Lock = Lock() - _operation_queue_thread : Thread = None - _operation_queue_thread_running : bool = False + _operation_queue: list[QueueOperation] = [] + _operation_queue_lock: Lock = Lock() + _operation_queue_thread: Thread = None + _operation_queue_thread_running: bool = False # configuration options/settings - _connection : mqtt.Connection = None - _queue_limit_size : int = 10 - _queue_limit_behavior : LimitBehavior = LimitBehavior.DROP_BACK - _queue_insert_behavior : InsertBehavior = InsertBehavior.INSERT_BACK - _incomplete_limit : int = 1 - _inflight_limit : int = 1 - _on_operation_sent_callback : FunctionType = None - _on_operation_sent_failure_callback : FunctionType = None - _on_operation_dropped_callback : FunctionType = None - _on_queue_full_callback : FunctionType = None - _on_queue_empty_callback : FunctionType = None - _queue_loop_time_sec : int = 1 - _enable_logging : bool = False - - def __init__(self, builder:MqttOperationQueueBuilder) -> None: + _connection: mqtt.Connection = None + _queue_limit_size: int = 10 + _queue_limit_behavior: LimitBehavior = LimitBehavior.DROP_BACK + _queue_insert_behavior: InsertBehavior = InsertBehavior.INSERT_BACK + _incomplete_limit: int = 1 + _inflight_limit: int = 1 + _on_operation_sent_callback: FunctionType = None + _on_operation_sent_failure_callback: FunctionType = None + _on_operation_dropped_callback: FunctionType = None + _on_queue_full_callback: FunctionType = None + _on_queue_empty_callback: FunctionType = None + _queue_loop_time_sec: int = 1 + _enable_logging: bool = False + + def __init__(self, builder: MqttOperationQueueBuilder) -> None: self._connection = builder._connection self._queue_limit_size = builder._queue_limit_size self._queue_limit_behavior = builder._queue_limit_behavior @@ -384,7 +390,7 @@ def __init__(self, builder:MqttOperationQueueBuilder) -> None: self._on_operation_dropped_callback = builder._on_operation_dropped_callback self._on_queue_full_callback = builder._on_queue_full_callback self._on_queue_empty_callback = builder._on_queue_empty_callback - self._queue_loop_time_sec = builder._queue_loop_time_ms / 1000.0 # convert to seconds since that is what sleep uses + self._queue_loop_time_sec = builder._queue_loop_time_ms / 1000.0 # convert to seconds since that is what sleep uses self._enable_logging = builder._enable_logging #################### @@ -400,7 +406,7 @@ def _add_operation_to_queue_insert(self, operation: QueueOperation) -> QueueResu Keyword Args: operation (QueueOperation): The operation to add. """ - result : QueueResult = QueueResult.SUCCESS + result: QueueResult = QueueResult.SUCCESS if (self._queue_insert_behavior == InsertBehavior.INSERT_FRONT): self._operation_queue.insert(0, operation) elif (self._queue_insert_behavior == InsertBehavior.INSERT_BACK): @@ -430,7 +436,7 @@ def _add_operation_to_queue_overflow(self, operation: QueueOperation) -> tuple: self._print_log_message(f"Dropped operation of type {result[1].type} from the front...") result[0] = self._add_operation_to_queue_insert(operation) elif (self._queue_limit_behavior == LimitBehavior.DROP_BACK): - end_of_queue = len(self._operation_queue)-1 + end_of_queue = len(self._operation_queue) - 1 result[1] = self._operation_queue[end_of_queue] del self._operation_queue[end_of_queue] self._print_log_message(f"Dropped operation of type {result[1].type} from the back...") @@ -446,10 +452,10 @@ def _add_operation_to_queue(self, operation: QueueOperation) -> QueueResult: Keyword Args: operation (QueueOperation): The operation to add. """ - result : QueueResult = QueueResult.SUCCESS - dropped_operation : QueueOperation = None + result: QueueResult = QueueResult.SUCCESS + dropped_operation: QueueOperation = None - if (operation == None): + if (operation is None): return QueueResult.ERROR_INVALID_ARGUMENT # CRITICAL SECTION @@ -459,7 +465,7 @@ def _add_operation_to_queue(self, operation: QueueOperation) -> QueueResult: if (self._queue_limit_size <= 0): result = self._add_operation_to_queue_insert(operation) else: - if (len(self._operation_queue)+1 <= self._queue_limit_size): + if (len(self._operation_queue) + 1 <= self._queue_limit_size): result = self._add_operation_to_queue_insert(operation) else: return_data = self._add_operation_to_queue_overflow(operation) @@ -473,18 +479,18 @@ def _add_operation_to_queue(self, operation: QueueOperation) -> QueueResult: if (result == QueueResult.SUCCESS): self._print_log_message(f"Added operation of type {operation.type} successfully to queue") - if (len(self._operation_queue) == self._queue_limit_size and dropped_operation == None): - if (self._on_queue_full_callback != None): + if (len(self._operation_queue) == self._queue_limit_size and dropped_operation is None): + if (self._on_queue_full_callback is not None): self._on_queue_full_callback() # Note: We invoke the dropped callback outside of the critical section to avoid deadlocks - if (dropped_operation != None): - if (self._on_operation_dropped_callback != None): + if (dropped_operation is not None): + if (self._on_operation_dropped_callback is not None): self._on_operation_dropped_callback(dropped_operation) return result - def _print_log_message(self, message:str) -> None: + def _print_log_message(self, message: str) -> None: """ Helper function: Prints to the console if logging is enabled. Just makes code a little cleaner and easier to process. @@ -492,7 +498,7 @@ def _print_log_message(self, message:str) -> None: Keyword Args: message (str): The message to print. """ - if self._enable_logging == True: + if self._enable_logging: print("[MqttOperationQueue] " + message) #################### @@ -504,7 +510,7 @@ def _perform_operation_publish(self, operation: QueueOperation) -> None: Helper function: Takes the publish operation and passes it to the MQTT connection. """ result = self._connection.publish(operation.topic, operation.payload, operation.qos, operation.retain) - if (self._on_operation_sent_callback != None): + if (self._on_operation_sent_callback is not None): self._on_operation_sent_callback(operation, result) def _perform_operation_subscribe(self, operation: QueueOperation) -> None: @@ -512,7 +518,7 @@ def _perform_operation_subscribe(self, operation: QueueOperation) -> None: Helper function: Takes the subscribe operation and passes it to the MQTT connection. """ result = self._connection.subscribe(operation.topic, operation.qos, operation.subscribe_callback) - if (self._on_operation_sent_callback != None): + if (self._on_operation_sent_callback is not None): self._on_operation_sent_callback(operation, result) def _perform_operation_unsubscribe(self, operation: QueueOperation) -> None: @@ -520,25 +526,25 @@ def _perform_operation_unsubscribe(self, operation: QueueOperation) -> None: Helper function: Takes the unsubscribe operation and passes it to the MQTT connection. """ result = self._connection.unsubscribe(operation.topic) - if (self._on_operation_sent_callback != None): + if (self._on_operation_sent_callback is not None): self._on_operation_sent_callback(operation, result) def _perform_operation_unknown(self, operation: QueueOperation) -> None: """ Helper function: Takes the operation if it is unknown and sends it as a failure to the callback. """ - if (operation == None): + if (operation is None): self._print_log_message("ERROR - got empty/none operation to perform") else: self._print_log_message("ERROR - got unknown operation to perform") - if (self._on_operation_sent_failure_callback != None): + if (self._on_operation_sent_failure_callback is not None): self._on_operation_sent_failure_callback(operation, QueueResult.UNKNOWN_OPERATION) def _perform_operation(self, operation: QueueOperation) -> None: """ Helper function: Based on the operation type, calls the appropriate helper function. """ - if (operation == None): + if (operation is None): self._perform_operation_unknown(operation) elif (operation.type == QueueOperationType.PUBLISH): self._perform_operation_publish(operation) @@ -559,11 +565,13 @@ def _check_operation_statistics(self) -> bool: statistics: mqtt.OperationStatisticsData = self._connection.get_stats() if (statistics.incomplete_operation_count >= self._incomplete_limit): if (self._incomplete_limit > 0): - self._print_log_message("Skipping running operation due to incomplete operation count being equal or higher than maximum") + self._print_log_message( + "Skipping running operation due to incomplete operation count being equal or higher than maximum") return False if (statistics.unacked_operation_count >= self._inflight_limit): if (self._inflight_limit > 0): - self._print_log_message("Skipping running operation due to inflight operation count being equal or higher than maximum") + self._print_log_message( + "Skipping running operation due to inflight operation count being equal or higher than maximum") return False return True @@ -578,14 +586,14 @@ def _run_operation(self) -> None: try: if (len(self._operation_queue) > 0): - operation : QueueOperation = self._operation_queue[0] + operation: QueueOperation = self._operation_queue[0] del self._operation_queue[0] self._print_log_message(f"Starting to perform operation of type {operation.type}") self._perform_operation(operation) if (len(self._operation_queue) <= 0): - if (self._on_queue_empty_callback != None): + if (self._on_queue_empty_callback is not None): self._on_queue_empty_callback() pass @@ -602,7 +610,7 @@ def _queue_loop(self) -> None: This function is called every queue_loop_time_ms milliseconds. This is where the logic for handling the queue resides. """ - while self._operation_queue_thread_running == True: + while self._operation_queue_thread_running: self._print_log_message("Performing operation loop...") if (self._check_operation_statistics()): self._run_operation() @@ -621,7 +629,7 @@ def start(self) -> None: the MQTT connection has the bandwidth for the next operation (based on incomplete_limit and inflight_limit) and, if the MQTT connection has bandwidth, will start a next operation from the queue. """ - if (self._operation_queue_thread != None): + if (self._operation_queue_thread is not None): self._print_log_message("Cannot start because queue is already started!") return self._operation_queue_thread = Thread(target=self._queue_loop) @@ -636,7 +644,7 @@ def stop(self) -> None: Note: calling stop() will block the thread temporarily as it waits for the operation queue thread to finish """ - if (self._operation_queue_thread == None): + if (self._operation_queue_thread is None): self._print_log_message("Cannot stop because queue is already stopped!") return self._operation_queue_thread_running = False @@ -730,13 +738,13 @@ def add_queue_operation(self, operation: QueueOperation) -> QueueResult: if (operation.type == QueueOperationType.NONE): return QueueResult.ERROR_INVALID_ARGUMENT elif (operation.type == QueueOperationType.PUBLISH): - if (operation.topic == None or operation.qos == None): + if (operation.topic is None or operation.qos is None): return QueueResult.ERROR_INVALID_ARGUMENT elif (operation.type == QueueOperationType.SUBSCRIBE): - if (operation.topic == None or operation.qos == None): + if (operation.topic is None or operation.qos is None): return QueueResult.ERROR_INVALID_ARGUMENT elif (operation.type == QueueOperationType.UNSUBSCRIBE): - if (operation.topic == None): + if (operation.topic is None): return QueueResult.ERROR_INVALID_ARGUMENT else: return QueueResult.UNKNOWN_ERROR @@ -748,7 +756,7 @@ def get_queue_size(self) -> int: """ # CRITICAL SECTION self._operation_queue_lock.acquire() - size : int = len(self._operation_queue) + size: int = len(self._operation_queue) self._operation_queue_lock.release() # END CRITICAL SECTION return size diff --git a/samples/operation_queue/mqtt_operation_queue_tests.py b/samples/operation_queue/mqtt_operation_queue_tests.py index b63fe089..5d70f413 100644 --- a/samples/operation_queue/mqtt_operation_queue_tests.py +++ b/samples/operation_queue/mqtt_operation_queue_tests.py @@ -12,34 +12,38 @@ TEST_TOPIC = "test/topic/" + str(uuid4()) PRINT_QUEUE_LOGS = False + class tester: connection: mqtt.Connection - on_queue_empty_future : Future = Future() - on_queue_sent_future : Future = Future() - on_queue_dropped_future : Future = Future() - cmdUtils : CommandLineUtils + on_queue_empty_future: Future = Future() + on_queue_sent_future: Future = Future() + on_queue_dropped_future: Future = Future() + cmdUtils: CommandLineUtils - def on_application_failure(self, test_name : str, error : Exception): - print (f"Error in test {test_name}: {error}") + def on_application_failure(self, test_name: str, error: Exception): + print(f"Error in test {test_name}: {error}") os._exit(-1) def test_connection_setup(self): self.connection = self.cmdUtils.build_mqtt_connection(None, None) - self.on_queue_empty_future : Future = Future() - self.on_queue_sent_future : Future = Future() - self.on_queue_dropped_future : Future = Future() + self.on_queue_empty_future: Future = Future() + self.on_queue_sent_future: Future = Future() + self.on_queue_dropped_future: Future = Future() - def test_operation_success(self, result : mqtt_operation_queue.QueueResult, test_name: str): + def test_operation_success(self, result: mqtt_operation_queue.QueueResult, test_name: str): if (result != mqtt_operation_queue.QueueResult.SUCCESS): - self.on_application_failure(test_name, ValueError(f"operation was not successful. Result: {result}", test_name)) + self.on_application_failure(test_name, ValueError( + f"operation was not successful. Result: {result}", test_name)) def test_connect_sub_pub_unsub(self): self.test_connection_setup() try: def on_queue_empty_future(): self.on_queue_empty_future.set_result(None) + def on_queue_sent_future(operation: mqtt_operation_queue.QueueOperation, _): self.on_queue_sent_future.set_result(operation) + def on_queue_dropped_future(operation: mqtt_operation_queue.QueueOperation): self.on_queue_dropped_future.set_result(operation) @@ -48,11 +52,21 @@ def on_queue_dropped_future(operation: mqtt_operation_queue.QueueOperation): queue_builder.with_on_queue_empty_callback(on_queue_empty_future) queue_builder.with_on_operation_sent_callback(on_queue_sent_future) queue_builder.with_on_operation_dropped_callback(on_queue_dropped_future) - mqtt_queue : mqtt_operation_queue.MqttOperationQueue = queue_builder.build() + mqtt_queue: mqtt_operation_queue.MqttOperationQueue = queue_builder.build() # Add the operations to the queue - self.test_operation_success(mqtt_queue.subscribe(TEST_TOPIC, mqtt.QoS.AT_LEAST_ONCE, None), "test_connect_sub_pub_unsub") - self.test_operation_success(mqtt_queue.publish(TEST_TOPIC, "hello_world", mqtt.QoS.AT_LEAST_ONCE), "test_connect_sub_pub_unsub") + self.test_operation_success( + mqtt_queue.subscribe( + TEST_TOPIC, + mqtt.QoS.AT_LEAST_ONCE, + None), + "test_connect_sub_pub_unsub") + self.test_operation_success( + mqtt_queue.publish( + TEST_TOPIC, + "hello_world", + mqtt.QoS.AT_LEAST_ONCE), + "test_connect_sub_pub_unsub") self.test_operation_success(mqtt_queue.unsubscribe(TEST_TOPIC), "test_connect_sub_pub_unsub") if (mqtt_queue.get_queue_size() != 3): @@ -63,17 +77,21 @@ def on_queue_dropped_future(operation: mqtt_operation_queue.QueueOperation): mqtt_queue.start() # Make sure the order is right. Order should be: Sub, Pub, Unsub - return_operation : mqtt_operation_queue.QueueOperation = self.on_queue_sent_future.result() - if (return_operation == None or return_operation.type != mqtt_operation_queue.QueueOperationType.SUBSCRIBE): - self.on_application_failure("test_connect_sub_pub_unsub", ValueError(f"First operation is not subscribe. Type is {return_operation.type}")) - self.on_queue_sent_future = Future() # reset future + return_operation: mqtt_operation_queue.QueueOperation = self.on_queue_sent_future.result() + if (return_operation is None or return_operation.type != mqtt_operation_queue.QueueOperationType.SUBSCRIBE): + self.on_application_failure("test_connect_sub_pub_unsub", ValueError( + f"First operation is not subscribe. Type is {return_operation.type}")) + self.on_queue_sent_future = Future() # reset future return_operation = self.on_queue_sent_future.result() - if (return_operation == None or return_operation.type != mqtt_operation_queue.QueueOperationType.PUBLISH): - self.on_application_failure("test_connect_sub_pub_unsub", ValueError(f"Second operation is not publish. Type is {return_operation.type}")) - self.on_queue_sent_future = Future() # reset future + if (return_operation is None or return_operation.type != mqtt_operation_queue.QueueOperationType.PUBLISH): + self.on_application_failure("test_connect_sub_pub_unsub", ValueError( + f"Second operation is not publish. Type is {return_operation.type}")) + self.on_queue_sent_future = Future() # reset future return_operation = self.on_queue_sent_future.result() - if (return_operation == None or return_operation.type != mqtt_operation_queue.QueueOperationType.UNSUBSCRIBE): - self.on_application_failure("test_connect_sub_pub_unsub", ValueError(f"Third operation is not unsubscribe. Type is {return_operation.type}")) + if (return_operation is None or return_operation.type != + mqtt_operation_queue.QueueOperationType.UNSUBSCRIBE): + self.on_application_failure("test_connect_sub_pub_unsub", ValueError( + f"Third operation is not unsubscribe. Type is {return_operation.type}")) self.on_queue_empty_future.result() mqtt_queue.stop() @@ -87,8 +105,10 @@ def test_drop_back(self): try: def on_queue_empty_future(): self.on_queue_empty_future.set_result(None) + def on_queue_sent_future(operation: mqtt_operation_queue.QueueOperation, _): self.on_queue_sent_future.set_result(operation) + def on_queue_dropped_future(operation: mqtt_operation_queue.QueueOperation): self.on_queue_dropped_future.set_result(operation) @@ -97,38 +117,53 @@ def on_queue_dropped_future(operation: mqtt_operation_queue.QueueOperation): queue_builder.with_on_queue_empty_callback(on_queue_empty_future) queue_builder.with_on_operation_sent_callback(on_queue_sent_future) queue_builder.with_on_operation_dropped_callback(on_queue_dropped_future) - queue_builder.with_queue_limit_size(2).with_queue_limit_behavior(mqtt_operation_queue.LimitBehavior.DROP_BACK) - mqtt_queue : mqtt_operation_queue.MqttOperationQueue = queue_builder.build() + queue_builder.with_queue_limit_size(2).with_queue_limit_behavior( + mqtt_operation_queue.LimitBehavior.DROP_BACK) + mqtt_queue: mqtt_operation_queue.MqttOperationQueue = queue_builder.build() # Add the operations to the queue - self.test_operation_success(mqtt_queue.subscribe(TEST_TOPIC, mqtt.QoS.AT_LEAST_ONCE, None), "test_drop_back") + self.test_operation_success( + mqtt_queue.subscribe( + TEST_TOPIC, + mqtt.QoS.AT_LEAST_ONCE, + None), + "test_drop_back") # Add 10 publishes for i in range(0, 10): - self.on_queue_dropped_future = Future() # reset future - self.test_operation_success(mqtt_queue.publish(TEST_TOPIC, "hello_world", mqtt.QoS.AT_LEAST_ONCE), "test_drop_back") + self.on_queue_dropped_future = Future() # reset future + self.test_operation_success( + mqtt_queue.publish( + TEST_TOPIC, + "hello_world", + mqtt.QoS.AT_LEAST_ONCE), + "test_drop_back") if (mqtt_queue.get_queue_size() != 2): self.on_application_failure("test_drop_back", ValueError("Queue size is not 2")) - self.on_queue_dropped_future = Future() # reset future + self.on_queue_dropped_future = Future() # reset future self.test_operation_success(mqtt_queue.unsubscribe(TEST_TOPIC), "test_drop_back") - dropped_operation : mqtt_operation_queue.QueueOperation = self.on_queue_dropped_future.result() - if (dropped_operation == None or dropped_operation.type != mqtt_operation_queue.QueueOperationType.PUBLISH): - self.on_application_failure("test_drop_back", ValueError(f"Dropped operation is not publish. Type is {dropped_operation.type}")) + dropped_operation: mqtt_operation_queue.QueueOperation = self.on_queue_dropped_future.result() + if (dropped_operation is None or dropped_operation.type != mqtt_operation_queue.QueueOperationType.PUBLISH): + self.on_application_failure("test_drop_back", ValueError( + f"Dropped operation is not publish. Type is {dropped_operation.type}")) connect_future = self.connection.connect() connect_future.result() mqtt_queue.start() # Make sure the order is right. Order should be: Sub, Unsub - return_operation : mqtt_operation_queue.QueueOperation = self.on_queue_sent_future.result() - if (return_operation == None or return_operation.type != mqtt_operation_queue.QueueOperationType.SUBSCRIBE): - self.on_application_failure("test_drop_back", ValueError(f"First operation is not subscribe. Type is {return_operation.type}")) - self.on_queue_sent_future = Future() # reset future + return_operation: mqtt_operation_queue.QueueOperation = self.on_queue_sent_future.result() + if (return_operation is None or return_operation.type != mqtt_operation_queue.QueueOperationType.SUBSCRIBE): + self.on_application_failure("test_drop_back", ValueError( + f"First operation is not subscribe. Type is {return_operation.type}")) + self.on_queue_sent_future = Future() # reset future return_operation = self.on_queue_sent_future.result() - if (return_operation == None or return_operation.type != mqtt_operation_queue.QueueOperationType.UNSUBSCRIBE): - self.on_application_failure("test_drop_back", ValueError(f"Second operation is not unsubscribe. Type is {return_operation.type}")) + if (return_operation is None or return_operation.type != + mqtt_operation_queue.QueueOperationType.UNSUBSCRIBE): + self.on_application_failure("test_drop_back", ValueError( + f"Second operation is not unsubscribe. Type is {return_operation.type}")) self.on_queue_empty_future.result() mqtt_queue.stop() @@ -146,27 +181,47 @@ def on_queue_dropped_future(operation: mqtt_operation_queue.QueueOperation): queue_builder = mqtt_operation_queue.MqttOperationQueueBuilder() queue_builder.with_connection(self.connection).with_enable_logging(PRINT_QUEUE_LOGS) queue_builder.with_on_operation_dropped_callback(on_queue_dropped_future) - queue_builder.with_queue_limit_size(2).with_queue_limit_behavior(mqtt_operation_queue.LimitBehavior.DROP_FRONT) - mqtt_queue : mqtt_operation_queue.MqttOperationQueue = queue_builder.build() + queue_builder.with_queue_limit_size(2).with_queue_limit_behavior( + mqtt_operation_queue.LimitBehavior.DROP_FRONT) + mqtt_queue: mqtt_operation_queue.MqttOperationQueue = queue_builder.build() # Add the operations to the queue - self.test_operation_success(mqtt_queue.subscribe(TEST_TOPIC, mqtt.QoS.AT_LEAST_ONCE, None), "test_drop_front") + self.test_operation_success( + mqtt_queue.subscribe( + TEST_TOPIC, + mqtt.QoS.AT_LEAST_ONCE, + None), + "test_drop_front") self.test_operation_success(mqtt_queue.unsubscribe(TEST_TOPIC), "test_drop_front") if (mqtt_queue.get_queue_size() != 2): self.on_application_failure("test_drop_front", ValueError("Queue size is not 2")) # Add two publishes, make sure drop order is correct - self.test_operation_success(mqtt_queue.publish(TEST_TOPIC, "hello_world", mqtt.QoS.AT_LEAST_ONCE), "test_drop_front") - dropped_operation : mqtt_operation_queue.QueueOperation = self.on_queue_dropped_future.result() - if (dropped_operation == None or dropped_operation.type != mqtt_operation_queue.QueueOperationType.SUBSCRIBE): - self.on_application_failure("test_drop_front", ValueError(f"First Dropped operation is not subscribe. Type is {dropped_operation.type}")) + self.test_operation_success( + mqtt_queue.publish( + TEST_TOPIC, + "hello_world", + mqtt.QoS.AT_LEAST_ONCE), + "test_drop_front") + dropped_operation: mqtt_operation_queue.QueueOperation = self.on_queue_dropped_future.result() + if (dropped_operation is None or dropped_operation.type != + mqtt_operation_queue.QueueOperationType.SUBSCRIBE): + self.on_application_failure("test_drop_front", ValueError( + f"First Dropped operation is not subscribe. Type is {dropped_operation.type}")) # second drop - self.on_queue_dropped_future = Future() # reset future - self.test_operation_success(mqtt_queue.publish(TEST_TOPIC, "hello_world", mqtt.QoS.AT_LEAST_ONCE), "test_drop_front") + self.on_queue_dropped_future = Future() # reset future + self.test_operation_success( + mqtt_queue.publish( + TEST_TOPIC, + "hello_world", + mqtt.QoS.AT_LEAST_ONCE), + "test_drop_front") dropped_operation = self.on_queue_dropped_future.result() - if (dropped_operation == None or dropped_operation.type != mqtt_operation_queue.QueueOperationType.UNSUBSCRIBE): - self.on_application_failure("test_drop_front", ValueError(f"First Dropped operation is not unsubscribe. Type is {dropped_operation.type}")) + if (dropped_operation is None or dropped_operation.type != + mqtt_operation_queue.QueueOperationType.UNSUBSCRIBE): + self.on_application_failure("test_drop_front", ValueError( + f"First Dropped operation is not unsubscribe. Type is {dropped_operation.type}")) except Exception as ex: self.on_application_failure("test_drop_front", ex) @@ -176,6 +231,7 @@ def test_add_front(self): try: def on_queue_empty_future(): self.on_queue_empty_future.set_result(None) + def on_queue_sent_future(operation: mqtt_operation_queue.QueueOperation, _): self.on_queue_sent_future.set_result(operation) @@ -183,20 +239,36 @@ def on_queue_sent_future(operation: mqtt_operation_queue.QueueOperation, _): queue_builder.with_connection(self.connection).with_enable_logging(PRINT_QUEUE_LOGS) queue_builder.with_on_queue_empty_callback(on_queue_empty_future) queue_builder.with_on_operation_sent_callback(on_queue_sent_future) - queue_builder.with_queue_limit_size(2).with_queue_limit_behavior(mqtt_operation_queue.LimitBehavior.DROP_BACK) + queue_builder.with_queue_limit_size(2).with_queue_limit_behavior( + mqtt_operation_queue.LimitBehavior.DROP_BACK) queue_builder.with_queue_insert_behavior(mqtt_operation_queue.InsertBehavior.INSERT_FRONT) - mqtt_queue : mqtt_operation_queue.MqttOperationQueue = queue_builder.build() + mqtt_queue: mqtt_operation_queue.MqttOperationQueue = queue_builder.build() # Fill with publishes - self.test_operation_success(mqtt_queue.publish(TEST_TOPIC, "hello_world", mqtt.QoS.AT_LEAST_ONCE), "test_add_front") - self.test_operation_success(mqtt_queue.publish(TEST_TOPIC, "hello_world", mqtt.QoS.AT_LEAST_ONCE), "test_add_front") + self.test_operation_success( + mqtt_queue.publish( + TEST_TOPIC, + "hello_world", + mqtt.QoS.AT_LEAST_ONCE), + "test_add_front") + self.test_operation_success( + mqtt_queue.publish( + TEST_TOPIC, + "hello_world", + mqtt.QoS.AT_LEAST_ONCE), + "test_add_front") if (mqtt_queue.get_queue_size() != 2): self.on_application_failure("test_add_front", ValueError("Queue size is not 2")) # Add unsubscribe than subscribe, which should result in the queue order of subscribe, unsubscribe self.test_operation_success(mqtt_queue.unsubscribe(TEST_TOPIC), "test_add_front") - self.test_operation_success(mqtt_queue.subscribe(TEST_TOPIC, mqtt.QoS.AT_LEAST_ONCE, None), "test_add_front") + self.test_operation_success( + mqtt_queue.subscribe( + TEST_TOPIC, + mqtt.QoS.AT_LEAST_ONCE, + None), + "test_add_front") if (mqtt_queue.get_queue_size() != 2): self.on_application_failure("test_add_front", ValueError("Queue size is not 2")) @@ -206,13 +278,16 @@ def on_queue_sent_future(operation: mqtt_operation_queue.QueueOperation, _): mqtt_queue.start() # Make sure the order is right. Order should be: Sub, Unsub - return_operation : mqtt_operation_queue.QueueOperation = self.on_queue_sent_future.result() - if (return_operation == None or return_operation.type != mqtt_operation_queue.QueueOperationType.SUBSCRIBE): - self.on_application_failure("test_add_front", ValueError(f"First operation is not subscribe. Type is {return_operation.type}")) - self.on_queue_sent_future = Future() # reset future + return_operation: mqtt_operation_queue.QueueOperation = self.on_queue_sent_future.result() + if (return_operation is None or return_operation.type != mqtt_operation_queue.QueueOperationType.SUBSCRIBE): + self.on_application_failure("test_add_front", ValueError( + f"First operation is not subscribe. Type is {return_operation.type}")) + self.on_queue_sent_future = Future() # reset future return_operation = self.on_queue_sent_future.result() - if (return_operation == None or return_operation.type != mqtt_operation_queue.QueueOperationType.UNSUBSCRIBE): - self.on_application_failure("test_add_front", ValueError(f"Second operation is not unsubscribe. Type is {return_operation.type}")) + if (return_operation is None or return_operation.type != + mqtt_operation_queue.QueueOperationType.UNSUBSCRIBE): + self.on_application_failure("test_add_front", ValueError( + f"Second operation is not unsubscribe. Type is {return_operation.type}")) self.on_queue_empty_future.result() mqtt_queue.stop() @@ -228,15 +303,17 @@ def test_add_error(self): try: queue_builder = mqtt_operation_queue.MqttOperationQueueBuilder() queue_builder.with_connection(self.connection).with_enable_logging(PRINT_QUEUE_LOGS) - queue_builder.with_queue_limit_size(2).with_queue_limit_behavior(mqtt_operation_queue.LimitBehavior.RETURN_ERROR) - mqtt_queue : mqtt_operation_queue.MqttOperationQueue = queue_builder.build() + queue_builder.with_queue_limit_size(2).with_queue_limit_behavior( + mqtt_operation_queue.LimitBehavior.RETURN_ERROR) + mqtt_queue: mqtt_operation_queue.MqttOperationQueue = queue_builder.build() # Fill with unsubscribe self.test_operation_success(mqtt_queue.unsubscribe(TEST_TOPIC), "test_add_error") self.test_operation_success(mqtt_queue.unsubscribe(TEST_TOPIC), "test_add_error") # Try to add another but it should return error stating queue is full - operation_result : mqtt_operation_queue.QueueResult = mqtt_queue.unsubscribe(TEST_TOPIC) + operation_result: mqtt_operation_queue.QueueResult = mqtt_queue.unsubscribe(TEST_TOPIC) if (operation_result != mqtt_operation_queue.QueueResult.ERROR_QUEUE_FULL): - self.on_application_failure("test_add_error", ValueError("Did not return queue full error trying to add operation to full queue")) + self.on_application_failure("test_add_error", ValueError( + "Did not return queue full error trying to add operation to full queue")) except Exception as ex: self.on_application_failure("test_add_error", ex) @@ -244,28 +321,28 @@ def test_add_error(self): def perform_tests(self): print("Starting test_connect_sub_pub_unsub test") self.test_connect_sub_pub_unsub() - print ("Finished test_connect_sub_pub_unsub test") + print("Finished test_connect_sub_pub_unsub test") print("Starting test_drop_back test") self.test_drop_back() - print ("Finished test_drop_back test") + print("Finished test_drop_back test") print("Starting test_drop_front test") self.test_drop_front() - print ("Finished test_drop_front test") + print("Finished test_drop_front test") print("Starting test_add_front test") self.test_add_front() - print ("Finished test_add_front test") + print("Finished test_add_front test") print("Starting test_add_error test") self.test_add_error() - print ("Finished test_add_error test") + print("Finished test_add_error test") -def perform_tests(cmdUtils : CommandLineUtils): +def perform_tests(cmdUtils: CommandLineUtils): tests = tester() tests.cmdUtils = cmdUtils tests.perform_tests() - print ("All tests finished. Exiting...") + print("All tests finished. Exiting...") os._exit(0) diff --git a/samples/operation_queue/operation_queue.py b/samples/operation_queue/operation_queue.py index f41183fd..8a684993 100644 --- a/samples/operation_queue/operation_queue.py +++ b/samples/operation_queue/operation_queue.py @@ -8,17 +8,15 @@ from concurrent.futures import Future # This sample uses the Message Broker for AWS IoT to send and receive messages -# through an MQTT connection. On startup, the device connects to the server, -# subscribes to a topic, and begins publishing messages to that topic. -# The device should receive those same messages back from the message broker, -# since it is subscribed to that same topic. +# through an MQTT connection, but uses a MQTT operation queue instead of +# directly using the MQTT connection to perform operations. import mqtt_operation_queue import mqtt_operation_queue_tests # Parse arguments from command_line_utils import CommandLineUtils -cmdUtils = CommandLineUtils("PubSub - Send and recieve messages through an MQTT connection.") +cmdUtils = CommandLineUtils("Operation Queue Sample") cmdUtils.add_common_mqtt_commands() cmdUtils.add_common_topic_message_commands() cmdUtils.add_common_proxy_commands() @@ -39,12 +37,13 @@ "If set to True (1 or greater), then the queue tests will be run instead of the sample (optional, default=0)", default=0, type=int) cmdUtils.register_command("is_ci", "", "If present the sample will run in CI mode (optional, default='None')") + # Needs to be called so the command utils parse the commands cmdUtils.get_args() received_count = 0 received_all_event = threading.Event() -is_ci = cmdUtils.get_command("is_ci", None) != None +is_ci = cmdUtils.get_command("is_ci", None) is not None if (cmdUtils.get_command("run_tests") > 0): mqtt_operation_queue_tests.perform_tests(cmdUtils) @@ -53,7 +52,6 @@ def on_connection_interrupted(connection, error, **kwargs): print("Connection interrupted. error: {}".format(error)) - # Callback when an interrupted connection is re-established. def on_connection_resumed(connection, return_code, session_present, **kwargs): print("Connection resumed. return_code: {} session_present: {}".format(return_code, session_present)) @@ -62,7 +60,6 @@ def on_connection_resumed(connection, return_code, session_present, **kwargs): print("Session did not persist. Resubscribing to existing topics...") connection.resubscribe_existing_topics() - # Callback when the subscribed topic receives a message def on_message_received(topic, payload, dup, qos, retain, **kwargs): print("Received message from topic '{}': {}".format(topic, payload)) @@ -75,10 +72,34 @@ def on_message_received(topic, payload, dup, qos, retain, **kwargs): # Callback when the mqtt operation queue is completely empty queue_empty_future = Future() def on_queue_empty(): - print ("Queue is completely empty!") + print("Queue is completely empty!") global queue_empty_future queue_empty_future.set_result(None) +# Callback when the mqtt operation queue is full +def on_queue_full(): + print("Operation queue is full and will start dropping operations should new operations come in") + +# Callback when the mqtt operation queue sends an operation +def on_queue_operation_sent(operation: mqtt_operation_queue.QueueOperation, _): + if (operation.type == mqtt_operation_queue.QueueOperationType.PUBLISH): + print(f"Sending publish with payload [{operation.payload}] from the operation queue") + else: + print(f"Sending operation of type {operation.type} from the operation queue") + +# Callback when the mqtt operation queue fails to send an operation +def on_queue_operation_sent_failure( + operation: mqtt_operation_queue.QueueOperation, + error: mqtt_operation_queue.QueueResult): + print(f"ERROR: Operation from queue with type {operation.type} failed with error {error}") + +# Callback when the mqtt operation queue drops an operation from the queue +def on_queue_operation_dropped(operation: mqtt_operation_queue.QueueOperation): + if (operation.type == mqtt_operation_queue.QueueOperationType.PUBLISH): + print(f"Publish with payload [{operation.payload}] dropped from the operation queue") + else: + print(f"Operation of type {operation.type} dropped from the operation queue") + if __name__ == '__main__': mqtt_connection = cmdUtils.build_mqtt_connection(on_connection_interrupted, on_connection_resumed) @@ -86,6 +107,10 @@ def on_queue_empty(): queue_builder = mqtt_operation_queue.MqttOperationQueueBuilder() queue_builder.with_connection(mqtt_connection).with_queue_limit_size(cmdUtils.get_command("queue_limit")) queue_builder.with_on_queue_empty_callback(on_queue_empty) + queue_builder.with_on_queue_full_callback(on_queue_full) + queue_builder.with_on_operation_sent_callback(on_queue_operation_sent) + queue_builder.with_on_operation_sent_failure_callback(on_queue_operation_sent_failure) + queue_builder.with_on_operation_dropped_callback(on_queue_operation_dropped) if (cmdUtils.get_command("queue_mode") == 0): queue_builder.with_queue_insert_behavior(mqtt_operation_queue.InsertBehavior.INSERT_BACK) queue_builder.with_queue_limit_behavior(mqtt_operation_queue.LimitBehavior.DROP_BACK) @@ -98,9 +123,9 @@ def on_queue_empty(): elif (cmdUtils.get_command("queue_mode") == 3): queue_builder.with_queue_insert_behavior(mqtt_operation_queue.InsertBehavior.INSERT_FRONT) queue_builder.with_queue_limit_behavior(mqtt_operation_queue.LimitBehavior.DROP_BACK) - mqtt_queue : mqtt_operation_queue.MqttOperationQueue = queue_builder.build() + mqtt_queue: mqtt_operation_queue.MqttOperationQueue = queue_builder.build() - if is_ci == False: + if not is_ci: print("Connecting to {} with client ID '{}'...".format( cmdUtils.get_command(cmdUtils.m_cmd_endpoint), cmdUtils.get_command("client_id"))) else: @@ -132,7 +157,7 @@ def on_queue_empty(): # Publish message to server desired number of times using the operation queue. # This step is skipped if message is blank. if message_string and message_count > 0: - print (f"Filling queue with {message_count} message(s)") + print(f"Filling queue with {message_count} message(s)") publish_count = 1 while (publish_count <= message_count) or (message_count == 0): @@ -146,10 +171,10 @@ def on_queue_empty(): publish_count += 1 # wait for the queue to be empty - print ("Waiting for all publishes in queue to be sent...") + print("Waiting for all publishes in queue to be sent...") queue_empty_future.result() else: - print ("Skipping sending publishes due to message being blank or message count being zero") + print("Skipping sending publishes due to message being blank or message count being zero") # Wait for all messages to be received and ACKs to be back from the server if message_count != 0 and not received_all_event.is_set(): From b41616966aaabbf67a922d0fefd0fa8ab048678d Mon Sep 17 00:00:00 2001 From: Noah Beard Date: Thu, 9 Mar 2023 16:00:00 -0500 Subject: [PATCH 06/12] Add running the sample to CI --- .github/workflows/ci.yml | 6 +++++ .../workflows/ci_run_operation_queue_cfg.json | 26 +++++++++++++++++++ .../ci_run_operation_queue_tests_cfg.json | 26 +++++++++++++++++++ 3 files changed, 58 insertions(+) create mode 100644 .github/workflows/ci_run_operation_queue_cfg.json create mode 100644 .github/workflows/ci_run_operation_queue_tests_cfg.json diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8de9f99e..16ed2666 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -245,6 +245,12 @@ jobs: export SOFTHSM2_CONF=/tmp/softhsm2.conf echo "directories.tokendir = /tmp/tokens" > /tmp/softhsm2.conf python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --file ${{ env.CI_SAMPLES_CFG_FOLDER }}/ci_run_pkcs11_connect_cfg.json + - name: run Operation Queue sample + run: | + python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --file ${{ env.CI_SAMPLES_CFG_FOLDER }}/ci_run_operation_queue_cfg.json + - name: run Operation Queue sample tests + run: | + python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --file ${{ env.CI_SAMPLES_CFG_FOLDER }}/ci_run_operation_queue_tests_cfg.json - name: configure AWS credentials (Cognito) uses: aws-actions/configure-aws-credentials@v1 with: diff --git a/.github/workflows/ci_run_operation_queue_cfg.json b/.github/workflows/ci_run_operation_queue_cfg.json new file mode 100644 index 00000000..151bf8de --- /dev/null +++ b/.github/workflows/ci_run_operation_queue_cfg.json @@ -0,0 +1,26 @@ +{ + "language": "Python", + "sample_file": "./aws-iot-device-sdk-python-v2/samples/operation_queue/operation_queue.py", + "sample_region": "us-east-1", + "sample_main_class": "", + "arguments": [ + { + "name": "--endpoint", + "secret": "ci/endpoint" + }, + { + "name": "--cert", + "secret": "ci/PubSub/cert", + "filename": "tmp_certificate.pem" + }, + { + "name": "--key", + "secret": "ci/PubSub/key", + "filename": "tmp_key.pem" + }, + { + "name": "--queue_mode", + "data": "1" + } + ] +} diff --git a/.github/workflows/ci_run_operation_queue_tests_cfg.json b/.github/workflows/ci_run_operation_queue_tests_cfg.json new file mode 100644 index 00000000..da2a132b --- /dev/null +++ b/.github/workflows/ci_run_operation_queue_tests_cfg.json @@ -0,0 +1,26 @@ +{ + "language": "Python", + "sample_file": "./aws-iot-device-sdk-python-v2/samples/operation_queue/operation_queue.py", + "sample_region": "us-east-1", + "sample_main_class": "", + "arguments": [ + { + "name": "--endpoint", + "secret": "ci/endpoint" + }, + { + "name": "--cert", + "secret": "ci/PubSub/cert", + "filename": "tmp_certificate.pem" + }, + { + "name": "--key", + "secret": "ci/PubSub/key", + "filename": "tmp_key.pem" + }, + { + "name": "--run_tests", + "data": "1" + } + ] +} From cd155e7d7949704818db2ea26e360b8a92138f70 Mon Sep 17 00:00:00 2001 From: Noah Beard Date: Fri, 10 Mar 2023 09:58:27 -0500 Subject: [PATCH 07/12] Work around the command_line_utils problem via dynamic symlink creation --- .gitignore | 5 ++++ .../mqtt_operation_queue_tests.py | 6 ++-- samples/operation_queue/operation_queue.py | 30 +++++++++++++++++-- .../command_line_utils.py | 1 - 4 files changed, 36 insertions(+), 6 deletions(-) rename samples/{operation_queue => utils}/command_line_utils.py (99%) diff --git a/.gitignore b/.gitignore index 5ba5e8af..1207efec 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,11 @@ __pycache__/ *.egg-info build/* +# ignore symlink command_line_utils.py in samples +samples/**/command_line_utils.py +# except for the utils folder - we want command_line_utils.py there +!samples/utils/command_line_utils.py + # docs are updated automatically by .github/workflows/docs.yml docs/ .vscode diff --git a/samples/operation_queue/mqtt_operation_queue_tests.py b/samples/operation_queue/mqtt_operation_queue_tests.py index 5d70f413..f0452738 100644 --- a/samples/operation_queue/mqtt_operation_queue_tests.py +++ b/samples/operation_queue/mqtt_operation_queue_tests.py @@ -7,7 +7,7 @@ import os import mqtt_operation_queue -from command_line_utils import CommandLineUtils +import command_line_utils TEST_TOPIC = "test/topic/" + str(uuid4()) PRINT_QUEUE_LOGS = False @@ -18,7 +18,7 @@ class tester: on_queue_empty_future: Future = Future() on_queue_sent_future: Future = Future() on_queue_dropped_future: Future = Future() - cmdUtils: CommandLineUtils + cmdUtils: command_line_utils.CommandLineUtils def on_application_failure(self, test_name: str, error: Exception): print(f"Error in test {test_name}: {error}") @@ -340,7 +340,7 @@ def perform_tests(self): print("Finished test_add_error test") -def perform_tests(cmdUtils: CommandLineUtils): +def perform_tests(cmdUtils: command_line_utils.CommandLineUtils): tests = tester() tests.cmdUtils = cmdUtils tests.perform_tests() diff --git a/samples/operation_queue/operation_queue.py b/samples/operation_queue/operation_queue.py index 8a684993..9ee1c64d 100644 --- a/samples/operation_queue/operation_queue.py +++ b/samples/operation_queue/operation_queue.py @@ -7,6 +7,33 @@ import json from concurrent.futures import Future +def link_command_line_utils(): + """ + An unfortunate necessary evil: We need to share the command_line_utils.py file across samples + but also need each sample in its own folder while still being directly executable. Python does + not like relative imports with non package/module python files, so we make a symlink if possible. + If it is not possible, we error out and direct where to find the file + """ + import os, pathlib + script_folder = pathlib.Path(os.path.abspath(__file__)).parent + command_line_utils_file = script_folder.joinpath("./command_line_utils.py") + # Do not create a symlink if the file already exists + if (command_line_utils_file.exists() and command_line_utils_file.is_file()): + return + # If the file does not exist, try to create a symlink to it + utils_command_line_utils_file = script_folder.parent.joinpath("./utils/command_line_utils.py") + if (utils_command_line_utils_file.exists() and utils_command_line_utils_file.is_file()): + command_line_utils_file.symlink_to(utils_command_line_utils_file, False) + print ("Created symlink to command_line_utils.py for sample to work correctly...") + else: + print("Error: Cannot find command_line_utils.py next to script nor in [../utils/command_line_utils.py]!") + print ("The sample cannot parse command line arguments without this file and therefore cannot run") + print ("Please place command_line_utils.py next to this script so it can run. You can find command_line_utils.py") + print ("at the following URL: https://github.com/aws/aws-iot-device-sdk-python-v2/blob/main/samples/utils/command_line_utils.py") + os._exit(1) +link_command_line_utils() +import command_line_utils + # This sample uses the Message Broker for AWS IoT to send and receive messages # through an MQTT connection, but uses a MQTT operation queue instead of # directly using the MQTT connection to perform operations. @@ -15,8 +42,7 @@ import mqtt_operation_queue_tests # Parse arguments -from command_line_utils import CommandLineUtils -cmdUtils = CommandLineUtils("Operation Queue Sample") +cmdUtils = command_line_utils.CommandLineUtils("Operation Queue Sample") cmdUtils.add_common_mqtt_commands() cmdUtils.add_common_topic_message_commands() cmdUtils.add_common_proxy_commands() diff --git a/samples/operation_queue/command_line_utils.py b/samples/utils/command_line_utils.py similarity index 99% rename from samples/operation_queue/command_line_utils.py rename to samples/utils/command_line_utils.py index dc8af5c2..b622bdd8 100644 --- a/samples/operation_queue/command_line_utils.py +++ b/samples/utils/command_line_utils.py @@ -5,7 +5,6 @@ from awscrt import io, http, auth from awsiot import mqtt_connection_builder, mqtt5_client_builder - class CommandLineUtils: def __init__(self, description) -> None: self.parser = argparse.ArgumentParser(description="Send and receive messages through and MQTT connection.") From 0a7ad895580397268ace19ecd1a3a49690f9cdef Mon Sep 17 00:00:00 2001 From: Noah Beard Date: Fri, 10 Mar 2023 12:18:04 -0500 Subject: [PATCH 08/12] Split all samples into their own folders, split samples README up into per sample READMEs --- .../workflows/ci_run_basic_connect_cfg.json | 2 +- .../workflows/ci_run_cognito_connect_cfg.json | 2 +- .../ci_run_custom_authorizer_connect_cfg.json | 2 +- .../ci_run_fleet_provisioning_cfg.json | 2 +- .github/workflows/ci_run_jobs_cfg.json | 2 +- .../ci_run_mqtt5_custom_authorizer_cfg.json | 2 +- ...qtt5_custom_authorizer_websockets_cfg.json | 2 +- .../ci_run_mqtt5_pkcs11_connect_cfg.json | 2 +- .../workflows/ci_run_mqtt5_pubsub_cfg.json | 2 +- .../workflows/ci_run_pkcs11_connect_cfg.json | 2 +- .github/workflows/ci_run_pubsub_cfg.json | 2 +- .github/workflows/ci_run_shadow_cfg.json | 2 +- .../ci_run_websocket_connect_cfg.json | 2 +- .../ci_run_windows_cert_connect_cfg.json | 2 +- codebuild/cd/test-prod-pypi.yml | 2 +- codebuild/cd/test-test-pypi.yml | 2 +- codebuild/samples/connect-linux.sh | 5 +- codebuild/samples/custom-auth-linux.sh | 2 +- codebuild/samples/pkcs11-connect-linux.sh | 2 +- codebuild/samples/pubsub-linux.sh | 2 +- codebuild/samples/pubsub-mqtt5-linux.sh | 4 +- codebuild/samples/shadow-linux.sh | 2 +- documents/FAQ.md | 10 +- samples/README.md | 936 +----------------- samples/basic_connect/README.md | 50 + samples/{ => basic_connect}/basic_connect.py | 28 +- samples/cognito_connect/README.md | 60 ++ .../{ => cognito_connect}/cognito_connect.py | 28 +- samples/command_line_utils.py | 403 -------- samples/custom_authorizer_connect/README.md | 47 + .../custom_authorizer_connect.py | 28 +- samples/discovery_greengrass/README.md | 9 + .../basic_discovery.py | 26 + samples/identity/README.md | 311 ++++++ .../fleet_provisioning.py} | 28 +- samples/ipc_greengrass/README.md | 99 ++ .../{ => ipc_greengrass}/ipc_greengrass.py | 40 - samples/jobs/README.md | 84 ++ samples/{ => jobs}/jobs.py | 28 +- .../mqtt5_custom_authorizer_connect/README.md | 72 ++ .../mqtt5_custom_authorizer_connect.py | 28 +- samples/mqtt5_pkcs11_connect/README.md | 113 +++ .../mqtt5_pkcs11_connect.py | 28 +- samples/mqtt5_pubsub/README.md | 75 ++ samples/{ => mqtt5_pubsub}/mqtt5_pubsub.py | 30 +- samples/operation_queue/README.md | 4 +- samples/pkcs11_connect/README.md | 109 ++ .../{ => pkcs11_connect}/pkcs11_connect.py | 28 +- samples/pubsub/README.md | 73 ++ samples/{ => pubsub}/pubsub.py | 28 +- samples/shadow/README.md | 87 ++ samples/{ => shadow}/shadow.py | 28 +- samples/websocket_connect/README.md | 45 + .../websocket_connect.py | 28 +- samples/windows_cert_connect/README.md | 104 ++ .../windows_cert_connect.py | 28 +- test/test_samples.py | 4 +- 57 files changed, 1781 insertions(+), 1395 deletions(-) create mode 100644 samples/basic_connect/README.md rename samples/{ => basic_connect}/basic_connect.py (61%) create mode 100644 samples/cognito_connect/README.md rename samples/{ => cognito_connect}/cognito_connect.py (63%) delete mode 100644 samples/command_line_utils.py create mode 100644 samples/custom_authorizer_connect/README.md rename samples/{ => custom_authorizer_connect}/custom_authorizer_connect.py (63%) create mode 100644 samples/discovery_greengrass/README.md rename samples/{ => discovery_greengrass}/basic_discovery.py (76%) create mode 100644 samples/identity/README.md rename samples/{fleetprovisioning.py => identity/fleet_provisioning.py} (90%) create mode 100644 samples/ipc_greengrass/README.md rename samples/{ => ipc_greengrass}/ipc_greengrass.py (64%) create mode 100644 samples/jobs/README.md rename samples/{ => jobs}/jobs.py (89%) create mode 100644 samples/mqtt5_custom_authorizer_connect/README.md rename samples/{ => mqtt5_custom_authorizer_connect}/mqtt5_custom_authorizer_connect.py (69%) create mode 100644 samples/mqtt5_pkcs11_connect/README.md rename samples/{ => mqtt5_pkcs11_connect}/mqtt5_pkcs11_connect.py (64%) create mode 100644 samples/mqtt5_pubsub/README.md rename samples/{ => mqtt5_pubsub}/mqtt5_pubsub.py (78%) create mode 100644 samples/pkcs11_connect/README.md rename samples/{ => pkcs11_connect}/pkcs11_connect.py (66%) create mode 100644 samples/pubsub/README.md rename samples/{ => pubsub}/pubsub.py (76%) create mode 100644 samples/shadow/README.md rename samples/{ => shadow}/shadow.py (90%) create mode 100644 samples/websocket_connect/README.md rename samples/{ => websocket_connect}/websocket_connect.py (59%) create mode 100644 samples/windows_cert_connect/README.md rename samples/{ => windows_cert_connect}/windows_cert_connect.py (64%) diff --git a/.github/workflows/ci_run_basic_connect_cfg.json b/.github/workflows/ci_run_basic_connect_cfg.json index fae564c2..4181c45b 100644 --- a/.github/workflows/ci_run_basic_connect_cfg.json +++ b/.github/workflows/ci_run_basic_connect_cfg.json @@ -1,6 +1,6 @@ { "language": "Python", - "sample_file": "./aws-iot-device-sdk-python-v2/samples/basic_connect.py", + "sample_file": "./aws-iot-device-sdk-python-v2/samples/basic_connect/basic_connect.py", "sample_region": "us-east-1", "sample_main_class": "", "arguments": [ diff --git a/.github/workflows/ci_run_cognito_connect_cfg.json b/.github/workflows/ci_run_cognito_connect_cfg.json index 1f6b6cd1..21db5bd5 100644 --- a/.github/workflows/ci_run_cognito_connect_cfg.json +++ b/.github/workflows/ci_run_cognito_connect_cfg.json @@ -1,6 +1,6 @@ { "language": "Python", - "sample_file": "./aws-iot-device-sdk-python-v2/samples/cognito_connect.py", + "sample_file": "./aws-iot-device-sdk-python-v2/samples/cognito_connect/cognito_connect.py", "sample_region": "us-east-1", "sample_main_class": "", "arguments": [ diff --git a/.github/workflows/ci_run_custom_authorizer_connect_cfg.json b/.github/workflows/ci_run_custom_authorizer_connect_cfg.json index cbd9afa9..b7ad1281 100644 --- a/.github/workflows/ci_run_custom_authorizer_connect_cfg.json +++ b/.github/workflows/ci_run_custom_authorizer_connect_cfg.json @@ -1,6 +1,6 @@ { "language": "Python", - "sample_file": "./aws-iot-device-sdk-python-v2/samples/custom_authorizer_connect.py", + "sample_file": "./aws-iot-device-sdk-python-v2/samples/custom_authorizer_connect/custom_authorizer_connect.py", "sample_region": "us-east-1", "sample_main_class": "", "arguments": [ diff --git a/.github/workflows/ci_run_fleet_provisioning_cfg.json b/.github/workflows/ci_run_fleet_provisioning_cfg.json index 0370102f..926361e8 100644 --- a/.github/workflows/ci_run_fleet_provisioning_cfg.json +++ b/.github/workflows/ci_run_fleet_provisioning_cfg.json @@ -1,6 +1,6 @@ { "language": "Python", - "sample_file": "./aws-iot-device-sdk-python-v2/samples/fleetprovisioning.py", + "sample_file": "./aws-iot-device-sdk-python-v2/samples/identity/fleet_provisioning.py", "sample_region": "us-east-1", "sample_main_class": "", "arguments": [ diff --git a/.github/workflows/ci_run_jobs_cfg.json b/.github/workflows/ci_run_jobs_cfg.json index bb7300d2..f34b7d8c 100644 --- a/.github/workflows/ci_run_jobs_cfg.json +++ b/.github/workflows/ci_run_jobs_cfg.json @@ -1,6 +1,6 @@ { "language": "Python", - "sample_file": "./aws-iot-device-sdk-python-v2/samples/jobs.py", + "sample_file": "./aws-iot-device-sdk-python-v2/samples/jobs/jobs.py", "sample_region": "us-east-1", "sample_main_class": "", "arguments": [ diff --git a/.github/workflows/ci_run_mqtt5_custom_authorizer_cfg.json b/.github/workflows/ci_run_mqtt5_custom_authorizer_cfg.json index f3608180..e4dff2ed 100644 --- a/.github/workflows/ci_run_mqtt5_custom_authorizer_cfg.json +++ b/.github/workflows/ci_run_mqtt5_custom_authorizer_cfg.json @@ -1,6 +1,6 @@ { "language": "Python", - "sample_file": "./aws-iot-device-sdk-python-v2/samples/mqtt5_custom_authorizer_connect.py", + "sample_file": "./aws-iot-device-sdk-python-v2/samples/mqtt5_custom_authorizer_connect/mqtt5_custom_authorizer_connect.py", "sample_region": "us-east-1", "sample_main_class": "", "arguments": [ diff --git a/.github/workflows/ci_run_mqtt5_custom_authorizer_websockets_cfg.json b/.github/workflows/ci_run_mqtt5_custom_authorizer_websockets_cfg.json index c77cbc12..fa0d3822 100644 --- a/.github/workflows/ci_run_mqtt5_custom_authorizer_websockets_cfg.json +++ b/.github/workflows/ci_run_mqtt5_custom_authorizer_websockets_cfg.json @@ -1,6 +1,6 @@ { "language": "Python", - "sample_file": "./aws-iot-device-sdk-python-v2/samples/mqtt5_custom_authorizer_connect.py", + "sample_file": "./aws-iot-device-sdk-python-v2/samples/mqtt5_custom_authorizer_connect/mqtt5_custom_authorizer_connect.py", "sample_region": "us-east-1", "sample_main_class": "", "arguments": [ diff --git a/.github/workflows/ci_run_mqtt5_pkcs11_connect_cfg.json b/.github/workflows/ci_run_mqtt5_pkcs11_connect_cfg.json index a8aaa3d9..899930ed 100644 --- a/.github/workflows/ci_run_mqtt5_pkcs11_connect_cfg.json +++ b/.github/workflows/ci_run_mqtt5_pkcs11_connect_cfg.json @@ -1,6 +1,6 @@ { "language": "Python", - "sample_file": "./aws-iot-device-sdk-python-v2/samples/mqtt5_pkcs11_connect.py", + "sample_file": "./aws-iot-device-sdk-python-v2/samples/mqtt5_pkcs11_connect/mqtt5_pkcs11_connect.py", "sample_region": "us-east-1", "sample_main_class": "", "arguments": [ diff --git a/.github/workflows/ci_run_mqtt5_pubsub_cfg.json b/.github/workflows/ci_run_mqtt5_pubsub_cfg.json index 28410e29..7d7b3d83 100644 --- a/.github/workflows/ci_run_mqtt5_pubsub_cfg.json +++ b/.github/workflows/ci_run_mqtt5_pubsub_cfg.json @@ -1,6 +1,6 @@ { "language": "Python", - "sample_file": "./aws-iot-device-sdk-python-v2/samples/mqtt5_pubsub.py", + "sample_file": "./aws-iot-device-sdk-python-v2/samples/mqtt5_pubsub/mqtt5_pubsub.py", "sample_region": "us-east-1", "sample_main_class": "mqtt5.pubsub.PubSub", "arguments": [ diff --git a/.github/workflows/ci_run_pkcs11_connect_cfg.json b/.github/workflows/ci_run_pkcs11_connect_cfg.json index dfcf9cdc..c19b053f 100644 --- a/.github/workflows/ci_run_pkcs11_connect_cfg.json +++ b/.github/workflows/ci_run_pkcs11_connect_cfg.json @@ -1,6 +1,6 @@ { "language": "Python", - "sample_file": "./aws-iot-device-sdk-python-v2/samples/pkcs11_connect.py", + "sample_file": "./aws-iot-device-sdk-python-v2/samples/pkcs11_connect/pkcs11_connect.py", "sample_region": "us-east-1", "sample_main_class": "", "arguments": [ diff --git a/.github/workflows/ci_run_pubsub_cfg.json b/.github/workflows/ci_run_pubsub_cfg.json index a10cbf34..a5bb717c 100644 --- a/.github/workflows/ci_run_pubsub_cfg.json +++ b/.github/workflows/ci_run_pubsub_cfg.json @@ -1,6 +1,6 @@ { "language": "Python", - "sample_file": "./aws-iot-device-sdk-python-v2/samples/pubsub.py", + "sample_file": "./aws-iot-device-sdk-python-v2/samples/pubsub/pubsub.py", "sample_region": "us-east-1", "sample_main_class": "", "arguments": [ diff --git a/.github/workflows/ci_run_shadow_cfg.json b/.github/workflows/ci_run_shadow_cfg.json index 01fd4824..1457dcbb 100644 --- a/.github/workflows/ci_run_shadow_cfg.json +++ b/.github/workflows/ci_run_shadow_cfg.json @@ -1,6 +1,6 @@ { "language": "Python", - "sample_file": "./aws-iot-device-sdk-python-v2/samples/shadow.py", + "sample_file": "./aws-iot-device-sdk-python-v2/samples/shadow/shadow.py", "sample_region": "us-east-1", "sample_main_class": "", "arguments": [ diff --git a/.github/workflows/ci_run_websocket_connect_cfg.json b/.github/workflows/ci_run_websocket_connect_cfg.json index d21980cd..0505ba7c 100644 --- a/.github/workflows/ci_run_websocket_connect_cfg.json +++ b/.github/workflows/ci_run_websocket_connect_cfg.json @@ -1,6 +1,6 @@ { "language": "Python", - "sample_file": "./aws-iot-device-sdk-python-v2/samples/websocket_connect.py", + "sample_file": "./aws-iot-device-sdk-python-v2/samples/websocket_connect/websocket_connect.py", "sample_region": "us-east-1", "sample_main_class": "", "arguments": [ diff --git a/.github/workflows/ci_run_windows_cert_connect_cfg.json b/.github/workflows/ci_run_windows_cert_connect_cfg.json index 2e16ae15..459fb888 100644 --- a/.github/workflows/ci_run_windows_cert_connect_cfg.json +++ b/.github/workflows/ci_run_windows_cert_connect_cfg.json @@ -1,6 +1,6 @@ { "language": "Python", - "sample_file": "./aws-iot-device-sdk-python-v2/samples/windows_cert_connect.py", + "sample_file": "./aws-iot-device-sdk-python-v2/samples/windows_cert_connect/windows_cert_connect.py", "sample_region": "us-east-1", "sample_main_class": "", "arguments": [ diff --git a/codebuild/cd/test-prod-pypi.yml b/codebuild/cd/test-prod-pypi.yml index 9777dc96..4bfb5737 100644 --- a/codebuild/cd/test-prod-pypi.yml +++ b/codebuild/cd/test-prod-pypi.yml @@ -28,7 +28,7 @@ phases: - CURRENT_TAG_VERSION=$(cat $CODEBUILD_SRC_DIR/VERSION) - python3 codebuild/cd/pip-install-with-retry.py --no-cache-dir --user awsiotsdk==$CURRENT_TAG_VERSION # Run PubSub sample - - python3 samples/pubsub.py --endpoint ${ENDPOINT} --cert /tmp/certificate.pem --key /tmp/privatekey.pem --ca_file /tmp/AmazonRootCA1.pem --verbosity Trace + - python3 samples/pubsub/pubsub.py --endpoint ${ENDPOINT} --cert /tmp/certificate.pem --key /tmp/privatekey.pem --ca_file /tmp/AmazonRootCA1.pem --verbosity Trace post_build: commands: diff --git a/codebuild/cd/test-test-pypi.yml b/codebuild/cd/test-test-pypi.yml index d202dcce..bb108906 100644 --- a/codebuild/cd/test-test-pypi.yml +++ b/codebuild/cd/test-test-pypi.yml @@ -29,7 +29,7 @@ phases: - python3 -m pip install typing - python3 codebuild/cd/pip-install-with-retry.py -i https://testpypi.python.org/simple --user awsiotsdk==$CURRENT_TAG_VERSION # Run PubSub sample - - python3 samples/pubsub.py --endpoint ${ENDPOINT} --cert /tmp/certificate.pem --key /tmp/privatekey.pem --ca_file /tmp/AmazonRootCA1.pem --verbosity Trace + - python3 samples/pubsub/pubsub.py --endpoint ${ENDPOINT} --cert /tmp/certificate.pem --key /tmp/privatekey.pem --ca_file /tmp/AmazonRootCA1.pem --verbosity Trace post_build: commands: diff --git a/codebuild/samples/connect-linux.sh b/codebuild/samples/connect-linux.sh index 5cecd2b9..f178eed4 100755 --- a/codebuild/samples/connect-linux.sh +++ b/codebuild/samples/connect-linux.sh @@ -5,13 +5,16 @@ set -o pipefail env -pushd $CODEBUILD_SRC_DIR/samples/ +pushd $CODEBUILD_SRC_DIR/samples/basic_connect ENDPOINT=$(aws secretsmanager get-secret-value --secret-id "ci/endpoint" --query "SecretString" | cut -f2 -d":" | sed -e 's/[\\\"\}]//g') echo "Basic Connect test" python3 basic_connect.py --endpoint $ENDPOINT --key /tmp/privatekey.pem --cert /tmp/certificate.pem +popd +pushd $CODEBUILD_SRC_DIR/samples/websocket_connect + echo "Websocket Connect test" python3 websocket_connect.py --endpoint $ENDPOINT --signing_region us-east-1 diff --git a/codebuild/samples/custom-auth-linux.sh b/codebuild/samples/custom-auth-linux.sh index 6fc7c2a2..bcb2af64 100755 --- a/codebuild/samples/custom-auth-linux.sh +++ b/codebuild/samples/custom-auth-linux.sh @@ -5,7 +5,7 @@ set -o pipefail env -pushd $CODEBUILD_SRC_DIR/samples/ +pushd $CODEBUILD_SRC_DIR/samples/custom_authorizer_connect/ ENDPOINT=$(aws secretsmanager get-secret-value --secret-id "ci/endpoint" --query "SecretString" | cut -f2 -d":" | sed -e 's/[\\\"\}]//g') AUTH_NAME=$(aws secretsmanager get-secret-value --secret-id "ci/CustomAuthorizer/name" --query "SecretString" | cut -f2 -d":" | sed -e 's/[\\\"\}]//g') diff --git a/codebuild/samples/pkcs11-connect-linux.sh b/codebuild/samples/pkcs11-connect-linux.sh index 48c2d0c9..616749cd 100755 --- a/codebuild/samples/pkcs11-connect-linux.sh +++ b/codebuild/samples/pkcs11-connect-linux.sh @@ -3,7 +3,7 @@ set -e set -o pipefail -pushd $CODEBUILD_SRC_DIR/samples/ +pushd $CODEBUILD_SRC_DIR/samples/pkcs11_connect/ ENDPOINT=$(aws secretsmanager get-secret-value --secret-id "ci/endpoint" --query "SecretString" | cut -f2 -d":" | sed -e 's/[\\\"\}]//g') diff --git a/codebuild/samples/pubsub-linux.sh b/codebuild/samples/pubsub-linux.sh index c7b5d797..49c60bc5 100755 --- a/codebuild/samples/pubsub-linux.sh +++ b/codebuild/samples/pubsub-linux.sh @@ -5,7 +5,7 @@ set -o pipefail env -pushd $CODEBUILD_SRC_DIR/samples/ +pushd $CODEBUILD_SRC_DIR/samples/pubsub/ ENDPOINT=$(aws secretsmanager get-secret-value --secret-id "ci/endpoint" --query "SecretString" | cut -f2 -d":" | sed -e 's/[\\\"\}]//g') diff --git a/codebuild/samples/pubsub-mqtt5-linux.sh b/codebuild/samples/pubsub-mqtt5-linux.sh index ee0e76b0..77c00ef4 100755 --- a/codebuild/samples/pubsub-mqtt5-linux.sh +++ b/codebuild/samples/pubsub-mqtt5-linux.sh @@ -5,11 +5,11 @@ set -o pipefail env -pushd $CODEBUILD_SRC_DIR/samples/ +pushd $CODEBUILD_SRC_DIR/samples/mqtt5_pubsub/ ENDPOINT=$(aws secretsmanager get-secret-value --secret-id "ci/endpoint" --query "SecretString" | cut -f2 -d":" | sed -e 's/[\\\"\}]//g') echo "MQTT5 PubSub test" python3 mqtt5_pubsub.py --endpoint $ENDPOINT --key /tmp/privatekey.pem --cert /tmp/certificate.pem -popd \ No newline at end of file +popd diff --git a/codebuild/samples/shadow-linux.sh b/codebuild/samples/shadow-linux.sh index 1fc1b54f..18b71ad1 100755 --- a/codebuild/samples/shadow-linux.sh +++ b/codebuild/samples/shadow-linux.sh @@ -5,7 +5,7 @@ set -o pipefail env -pushd $CODEBUILD_SRC_DIR/samples/ +pushd $CODEBUILD_SRC_DIR/samples/shadow/ ENDPOINT=$(aws secretsmanager get-secret-value --secret-id "ci/endpoint" --query "SecretString" | cut -f2 -d":" | sed -e 's/[\\\"\}]//g') diff --git a/documents/FAQ.md b/documents/FAQ.md index b981b292..ab358757 100644 --- a/documents/FAQ.md +++ b/documents/FAQ.md @@ -40,7 +40,7 @@ Please note that on Mac, once a private key is used with a certificate, that cer static: certificate has an existing certificate-key pair that was previously imported into the Keychain. Using key from Keychain instead of the one provided. ``` -### How do debug in VSCode? +### How do debug in VSCode? Here is an example launch.json file to run the pubsub sample ``` json @@ -54,7 +54,7 @@ Here is an example launch.json file to run the pubsub sample "name": "PubSub", "type": "python", "request": "launch", - "program": "${workspaceFolder}/samples/pubsub.py", + "program": "${workspaceFolder}/samples/pubsub/pubsub.py", "args": [ "--endpoint", "-ats.iot..amazonaws.com", "--ca_file", "", @@ -79,13 +79,13 @@ Here is an example launch.json file to run the pubsub sample * Device certificate * Intermediate device certificate that is used to generate the key below * When using samples it can look like this: `--cert abcde12345-certificate.pem.crt` - * Key files + * Key files * You should have generated/downloaded private and public keys that will be used to verify that communications are coming from you * When using samples you only need the private key and it will look like this: `--key abcde12345-private.pem.key` ### I still have more questions about the this sdk? * [Here](https://docs.aws.amazon.com/iot/latest/developerguide/what-is-aws-iot.html) are the AWS IoT Core docs for more details about IoT Core -* [Here](https://docs.aws.amazon.com/greengrass/v2/developerguide/what-is-iot-greengrass.html) are the AWS IoT Greengrass v2 docs for more details about greengrass +* [Here](https://docs.aws.amazon.com/greengrass/v2/developerguide/what-is-iot-greengrass.html) are the AWS IoT Greengrass v2 docs for more details about greengrass * [Discussion](https://github.com/aws/aws-iot-device-sdk-python-v2/discussions) questions are also a great way to ask other questions about this sdk. -* [Open an issue](https://github.com/aws/aws-iot-device-sdk-python-v2/issues) if you find a bug or have a feature request \ No newline at end of file +* [Open an issue](https://github.com/aws/aws-iot-device-sdk-python-v2/issues) if you find a bug or have a feature request diff --git a/samples/README.md b/samples/README.md index 6344ee3c..2c96f872 100644 --- a/samples/README.md +++ b/samples/README.md @@ -1,926 +1,50 @@ # Sample apps for the AWS IoT Device SDK v2 for Python -* [MQTT5 PubSub](#mqtt5-pubsub) -* [PubSub](#pubsub) -* [Basic Connect](#basic-connect) -* [Websocket Connect](#websocket-connect) -* [PKCS#11 Connect](#pkcs11-connect) -* [Windows Certificate Connect](#windows-certificate-connect) -* [Custom Authorizer Connect](#custom-authorizer-connect) -* [Cognito Connect](#cognito-connect) -* [Shadow](#shadow) -* [Jobs](#jobs) -* [Fleet Provisioning](#fleet-provisioning) -* [Greengrass Discovery](#greengrass-discovery) +* [MQTT5 PubSub](./mqtt5_pubsub/README.md) +* [PubSub](./pubsub/README.md) +* [Basic Connect](./basic_connect/README.md) +* [Websocket Connect](./websocket_connect/README.md) +* [MQTT5 PKCS#11 Connect](./mqtt5_pkcs11_connect/README.md) +* [PKCS#11 Connect](./pkcs11_connect/README.md) +* [Windows Certificate Connect](./windows_cert_connect/README.md) +* [MQTT5 Custom Authorizer Connect](./mqtt5_custom_authorizer_connect/README.md) +* [Custom Authorizer Connect](./custom_authorizer_connect/README.md) +* [Cognito Connect](./cognito_connect/README.md) +* [Shadow](./shadow/README.md) +* [Jobs](./jobs/README.md) +* [Fleet Provisioning](./identity/README.md) +* [Greengrass Discovery](./discovery_greengrass/README.md) +* [Greengrass IPC](./ipc_greengrass/README.md) -## Build instructions +### Build instructions -First, install the aws-iot-devices-sdk-python-v2 with following the instructions from [Installation](../README.md#Installation). +First, install the `aws-iot-devices-sdk-python-v2` with following the instructions from [Installation](../README.md#Installation). -Then change into the samples directory to run the Python commands to execute the samples. You can view the commands of a sample like this: +Then change into the `samples` folder/directory to run the Python commands to execute the samples. Each sample README has instructions on how to run each sample and each sample can be run from the `samples` folder. For example, to run the [PubSub](./pubsub/README.md) sample: ``` sh -# For Windows: replace 'python3' with 'python' -python3 pubsub.py --help +# For Windows: replace 'python3' with 'python' and '/' with '\' +python3 pubsub/pubsub.py --endpoint --cert --key ``` -## MQTT5 PubSub -This sample uses the -[Message Broker](https://docs.aws.amazon.com/iot/latest/developerguide/iot-message-broker.html) -for AWS IoT to send and receive messages -through an MQTT5 connection. +### Sample Help -MQTT5 introduces additional features and enhancements that improve the development experience with MQTT. You can read more about MQTT5 in the Python V2 SDK by checking out the [MQTT5 user guide](../documents/MQTT5.md). +All samples will show their options by passing in `--help`. For example: -Note: MQTT5 support is currently in **developer preview**. We encourage feedback at all times, but feedback during the preview window is especially valuable in shaping the final product. During the preview period we may make backwards-incompatible changes to the public API, but in general, this is something we will try our best to avoid. - -On startup, the device connects to the server, -subscribes to a topic, and begins publishing messages to that topic. -The device should receive those same messages back from the message broker, -since it is subscribed to that same topic. -Status updates are continually printed to the console. - -Source: `samples/mqtt5_pubsub.py` - -Your Thing's [Policy](https://docs.aws.amazon.com/iot/latest/developerguide/iot-policies.html) must provide privileges for this sample to connect, subscribe, publish, and receive. Make sure your policy allows a client ID of `test-*` to connect or use `--client_id ` to send the client ID your policy supports. - -
-(see sample policy) -
-{
-  "Version": "2012-10-17",
-  "Statement": [
-    {
-      "Effect": "Allow",
-      "Action": [
-        "iot:Publish",
-        "iot:Receive"
-      ],
-      "Resource": [
-        "arn:aws:iot:region:account:topic/test/topic"
-      ]
-    },
-    {
-      "Effect": "Allow",
-      "Action": [
-        "iot:Subscribe"
-      ],
-      "Resource": [
-        "arn:aws:iot:region:account:topicfilter/test/topic"
-      ]
-    },
-    {
-      "Effect": "Allow",
-      "Action": [
-        "iot:Connect"
-      ],
-      "Resource": [
-        "arn:aws:iot:region:account:client/test-*"
-      ]
-    }
-  ]
-}
-
-
- -Run the sample like this: -``` sh -# For Windows: replace 'python3' with 'python' -python3 mqtt5_pubsub.py --endpoint --ca_file --cert --key -``` - -## PubSub - -This sample uses the -[Message Broker](https://docs.aws.amazon.com/iot/latest/developerguide/iot-message-broker.html) -for AWS IoT to send and receive messages -through an MQTT connection. On startup, the device connects to the server, -subscribes to a topic, and begins publishing messages to that topic. -The device should receive those same messages back from the message broker, -since it is subscribed to that same topic. -Status updates are continually printed to the console. - -Source: `samples/pubsub.py` - -Your Thing's [Policy](https://docs.aws.amazon.com/iot/latest/developerguide/iot-policies.html) must provide privileges for this sample to connect, subscribe, publish, and receive. Make sure your policy allows a client ID of `test-*` to connect or use `--client_id ` to send the client ID your policy supports. - -
-(see sample policy) -
-{
-  "Version": "2012-10-17",
-  "Statement": [
-    {
-      "Effect": "Allow",
-      "Action": [
-        "iot:Publish",
-        "iot:Receive"
-      ],
-      "Resource": [
-        "arn:aws:iot:region:account:topic/test/topic"
-      ]
-    },
-    {
-      "Effect": "Allow",
-      "Action": [
-        "iot:Subscribe"
-      ],
-      "Resource": [
-        "arn:aws:iot:region:account:topicfilter/test/topic"
-      ]
-    },
-    {
-      "Effect": "Allow",
-      "Action": [
-        "iot:Connect"
-      ],
-      "Resource": [
-        "arn:aws:iot:region:account:client/test-*"
-      ]
-    }
-  ]
-}
-
-
- -Run the sample like this: -``` sh -# For Windows: replace 'python3' with 'python' -python3 pubsub.py --endpoint --ca_file --cert --key -``` - -## Basic Connect - -This sample makes an MQTT connection using a certificate and key file. On startup, the device connects to the server using the certificate and key files, and then disconnects. -This sample is for reference on connecting via certificate and key files. - -Source: `samples/basic_connect.py` - -Your Thing's [Policy](https://docs.aws.amazon.com/iot/latest/developerguide/iot-policies.html) must provide privileges for this sample to connect. Make sure your policy allows a client ID of `test-*` to connect or use `--client_id ` to send the client ID your policy supports. - -
-(see sample policy) -
-{
-  "Version": "2012-10-17",
-  "Statement": [
-    {
-      "Effect": "Allow",
-      "Action": [
-        "iot:Connect"
-      ],
-      "Resource": [
-        "arn:aws:iot:region:account:client/test-*"
-      ]
-    }
-  ]
-}
-
-
- -Run the sample like this: -``` sh -# For Windows: replace 'python3' with 'python' -python3 basic_connect.py --endpoint --ca_file --cert --key -``` - -## Websocket Connect - -This sample makes an MQTT connection via websockets and then disconnects. On startup, the device connects to the server via websockets and then disconnects. -This sample is for reference on connecting via websockets. - -Source: `samples/websocket_connect.py` - -Your Thing's [Policy](https://docs.aws.amazon.com/iot/latest/developerguide/iot-policies.html) must provide privileges for this sample to connect. Make sure your policy allows a client ID of `test-*` to connect or use `--client_id ` to send the client ID your policy supports. - -
-(see sample policy) -
-{
-  "Version": "2012-10-17",
-  "Statement": [
-    {
-      "Effect": "Allow",
-      "Action": [
-        "iot:Connect"
-      ],
-      "Resource": [
-        "arn:aws:iot:region:account:client/test-*"
-      ]
-    }
-  ]
-}
-
-
- -Run the sample like this: -``` sh -# For Windows: replace 'python3' with 'python' -python3 websocket_connect.py --endpoint --ca_file --signing_region -``` - -Note that using Websockets will attempt to fetch the AWS credentials from your enviornment variables or local files. See the [authorizing direct AWS](https://docs.aws.amazon.com/iot/latest/developerguide/authorizing-direct-aws.html) page for documentation on how to get the AWS credentials, which then you can set to the `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS`, and `AWS_SESSION_TOKEN` environment variables. - -## PKCS#11 Connect - -This sample is similar to the [Basic Connect](#basic-connect), -but the private key for mutual TLS is stored on a PKCS#11 compatible smart card or Hardware Security Module (HSM) - -WARNING: Unix only. Currently, TLS integration with PKCS#11 is only available on Unix devices. - -source: `samples/pkcs11_connect.py` - -Your Thing's [Policy](https://docs.aws.amazon.com/iot/latest/developerguide/iot-policies.html) must provide privileges for this sample to connect. Make sure your policy allows a client ID of `test-*` to connect or use `--client_id ` to send the client ID your policy supports. - -
-(see sample policy) -
-{
-  "Version": "2012-10-17",
-  "Statement": [
-    {
-      "Effect": "Allow",
-      "Action": [
-        "iot:Connect"
-      ],
-      "Resource": [
-        "arn:aws:iot:region:account:client/test-*"
-      ]
-    }
-  ]
-}
-
-
- -To run this sample using [SoftHSM2](https://www.opendnssec.org/softhsm/) as the PKCS#11 device: - -1) Create an IoT Thing with a certificate and key if you haven't already. - -2) Convert the private key into PKCS#8 format - ```sh - openssl pkcs8 -topk8 -in -out -nocrypt - ``` - -3) Install [SoftHSM2](https://www.opendnssec.org/softhsm/): - ```sh - sudo apt install softhsm - ``` - - Check that it's working: - ```sh - softhsm2-util --show-slots - ``` - - If this spits out an error message, create a config file: - * Default location: `~/.config/softhsm2/softhsm2.conf` - * This file must specify token dir, default value is: - ``` - directories.tokendir = /usr/local/var/lib/softhsm/tokens/ - ``` - -4) Create token and import private key. - - You can use any values for the labels, PINs, etc - ```sh - softhsm2-util --init-token --free --label --pin --so-pin - ``` - - Note which slot the token ended up in - - ```sh - softhsm2-util --import --slot --label --id --pin - ``` - -5) Now you can run the sample: - ```sh - # For Windows: replace 'python3' with 'python' - python3 pkcs11_connect.py --endpoint --ca_file --cert --pkcs11_lib --pin --token_label --key_label - ``` - -## Windows Certificate Connect - -WARNING: Windows only - -This sample is similar to the basic [Connect](#basic-connect), -but your certificate and private key are in a -[Windows certificate store](https://docs.microsoft.com/en-us/windows-hardware/drivers/install/certificate-stores), -rather than simply being files on disk. - -To run this sample you need the path to your certificate in the store, -which will look something like: -"CurrentUser\My\A11F8A9B5DF5B98BA3508FBCA575D09570E0D2C6" -(where "CurrentUser\My" is the store and "A11F8A9B5DF5B98BA3508FBCA575D09570E0D2C6" is the certificate's thumbprint) - -If your certificate and private key are in a -[TPM](https://docs.microsoft.com/en-us/windows/security/information-protection/tpm/trusted-platform-module-overview),, -you would use them by passing their certificate store path. - -source: `samples/windows_cert_connect.py` - -Your Thing's [Policy](https://docs.aws.amazon.com/iot/latest/developerguide/iot-policies.html) must provide privileges for this sample to connect. Make sure your policy allows a client ID of `test-*` to connect or use `--client_id ` to send the client ID your policy supports. - -
-(see sample policy) -
-{
-  "Version": "2012-10-17",
-  "Statement": [
-    {
-      "Effect": "Allow",
-      "Action": [
-        "iot:Connect"
-      ],
-      "Resource": [
-        "arn:aws:iot:region:account:client/test-*"
-      ]
-    }
-  ]
-}
-
-
- -To run this sample with a basic certificate from AWS IoT Core: - -1) Create an IoT Thing with a certificate and key if you haven't already. - -2) Combine the certificate and private key into a single .pfx file. - - You will be prompted for a password while creating this file. Remember it for the next step. - - If you have OpenSSL installed: - ```powershell - openssl pkcs12 -in certificate.pem.crt -inkey private.pem.key -out certificate.pfx - ``` - - Otherwise use [CertUtil](https://docs.microsoft.com/en-us/windows-server/administration/windows-commands/certutil). - ```powershell - certutil -mergePFX certificate.pem.crt,private.pem.key certificate.pfx - ``` - -3) Add the .pfx file to a Windows certificate store using PowerShell's - [Import-PfxCertificate](https://docs.microsoft.com/en-us/powershell/module/pki/import-pfxcertificate) - - In this example we're adding it to "CurrentUser\My" - - ```powershell - $mypwd = Get-Credential -UserName 'Enter password below' -Message 'Enter password below' - Import-PfxCertificate -FilePath certificate.pfx -CertStoreLocation Cert:\CurrentUser\My -Password $mypwd.Password - ``` - - Note the certificate thumbprint that is printed out: - ``` - Thumbprint Subject - ---------- ------- - A11F8A9B5DF5B98BA3508FBCA575D09570E0D2C6 CN=AWS IoT Certificate - ``` - - So this certificate's path would be: "CurrentUser\My\A11F8A9B5DF5B98BA3508FBCA575D09570E0D2C6" - -4) Now you can run the sample: - - ```sh - # For Windows: replace 'python3' with 'python' - python3 windows_cert_connect.py --endpoint --ca_file --cert - ``` - -## Custom Authorizer Connect - -This sample makes an MQTT connection and connects through a [Custom Authorizer](https://docs.aws.amazon.com/iot/latest/developerguide/custom-authentication.html). On startup, the device connects to the server and then disconnects. This sample is for reference on connecting using a custom authorizer. - -Your Thing's [Policy](https://docs.aws.amazon.com/iot/latest/developerguide/iot-policies.html) must provide privileges for this sample to connect. Make sure your policy allows a client ID of `test-*` to connect or use `--client_id ` to send the client ID your policy supports. - -
-(see sample policy) -
-{
-  "Version": "2012-10-17",
-  "Statement": [
-    {
-      "Effect": "Allow",
-      "Action": [
-        "iot:Connect"
-      ],
-      "Resource": [
-        "arn:aws:iot:region:account:client/test-*"
-      ]
-    }
-  ]
-}
-
-
- -Run the sample like this: -``` sh -# For Windows: replace 'python3' with 'python' -python3 custom_authorizer_connect.py --endpoint --custom_auth_authorizer_name -``` - -You will need to setup your Custom Authorizer so that the lambda function returns a policy document. See [this page on the documentation](https://docs.aws.amazon.com/iot/latest/developerguide/config-custom-auth.html) for more details and example return result. - -## Cognito Connect - -This sample makes an MQTT websocket connection and connects through a [Cognito](https://aws.amazon.com/cognito/) identity. On startup, the device connects to the server and then disconnects. This sample is for reference on connecting using Cognito. - -To run this sample, you need to have a Cognito identifier ID. You can get a Cognito identifier ID by creating a Cognito identity pool. For creating Cognito identity pools, please see the following page on the AWS documentation: [Tutorial: Creating an identity pool](https://docs.aws.amazon.com/cognito/latest/developerguide/tutorial-create-identity-pool.html) - -**Note:** This sample assumes using an identity pool with unauthenticated identity access for the sake of convenience. Please follow best practices in a real world application based on the needs of your application and the intended use case. - -Once you have a Cognito identity pool, you can run the following CLI command to get the Cognito identity pool ID: -```sh -aws cognito-identity get-id --identity-pool-id -# result from above command -{ - "IdentityId": "" -} -``` - -You can then use the returned ID in the `IdentityId` result as the input for the `--cognito_identity` argument. Please note that the Cognito identity pool ID is **not** the same as a Cognito identity ID and the sample will not work if you pass a Cognito pool id. - -Your IoT Thing's [Policy](https://docs.aws.amazon.com/iot/latest/developerguide/iot-policies.html) must provide privileges for this sample to connect. Make sure your policy allows a client ID of `test-*` to connect or use `--client_id ` to send the client ID your policy supports. - -
-(see sample policy) -
-{
-  "Version": "2012-10-17",
-  "Statement": [
-    {
-      "Effect": "Allow",
-      "Action": [
-        "iot:Connect"
-      ],
-      "Resource": [
-        "arn:aws:iot:region:account:client/test-*"
-      ]
-    }
-  ]
-}
-
-
- -Run the sample like this: -``` sh -# For Windows: replace 'python3' with 'python' -python3 cognito_connect.py --endpoint --signing_region --cognito_identity -``` - -## Shadow - -This sample uses the AWS IoT -[Device Shadow](https://docs.aws.amazon.com/iot/latest/developerguide/iot-device-shadows.html) -Service to keep a property in -sync between device and server. Imagine a light whose color may be changed -through an app, or set by a local user. - -Once connected, type a value in the terminal and press Enter to update -the property's "reported" value. The sample also responds when the "desired" -value changes on the server. To observe this, edit the Shadow document in -the AWS Console and set a new "desired" value. - -On startup, the sample requests the shadow document to learn the property's -initial state. The sample also subscribes to "delta" events from the server, -which are sent when a property's "desired" value differs from its "reported" -value. When the sample learns of a new desired value, that value is changed -on the device and an update is sent to the server with the new "reported" -value. - -Source: `samples/shadow.py` - -Run the sample like this: -``` sh -# For Windows: replace 'python3' with 'python' -python3 shadow.py --endpoint --ca_file --cert --key --thing_name -``` - -Your Thing's [Policy](https://docs.aws.amazon.com/iot/latest/developerguide/iot-policies.html) must provide privileges for this sample to connect, subscribe, publish, and receive. Make sure your policy allows a client ID of `test-*` to connect or use `--client_id ` to send the client ID your policy supports. - -
-(see sample policy) -
-{
-  "Version": "2012-10-17",
-  "Statement": [
-    {
-      "Effect": "Allow",
-      "Action": [
-        "iot:Publish"
-      ],
-      "Resource": [
-        "arn:aws:iot:region:account:topic/$aws/things/thingname/shadow/get",
-        "arn:aws:iot:region:account:topic/$aws/things/thingname/shadow/update"
-      ]
-    },
-    {
-      "Effect": "Allow",
-      "Action": [
-        "iot:Receive"
-      ],
-      "Resource": [
-        "arn:aws:iot:region:account:topic/$aws/things/thingname/shadow/get/accepted",
-        "arn:aws:iot:region:account:topic/$aws/things/thingname/shadow/get/rejected",
-        "arn:aws:iot:region:account:topic/$aws/things/thingname/shadow/update/accepted",
-        "arn:aws:iot:region:account:topic/$aws/things/thingname/shadow/update/rejected",
-        "arn:aws:iot:region:account:topic/$aws/things/thingname/shadow/update/delta"
-      ]
-    },
-    {
-      "Effect": "Allow",
-      "Action": [
-        "iot:Subscribe"
-      ],
-      "Resource": [
-        "arn:aws:iot:region:account:topicfilter/$aws/things/thingname/shadow/get/accepted",
-        "arn:aws:iot:region:account:topicfilter/$aws/things/thingname/shadow/get/rejected",
-        "arn:aws:iot:region:account:topicfilter/$aws/things/thingname/shadow/update/accepted",
-        "arn:aws:iot:region:account:topicfilter/$aws/things/thingname/shadow/update/rejected",
-        "arn:aws:iot:region:account:topicfilter/$aws/things/thingname/shadow/update/delta"
-      ]
-    },
-    {
-      "Effect": "Allow",
-      "Action": "iot:Connect",
-      "Resource": "arn:aws:iot:region:account:client/test-*"
-    }
-  ]
-}
-
-
- -## Jobs - -This sample uses the AWS IoT -[Jobs](https://docs.aws.amazon.com/iot/latest/developerguide/iot-jobs.html) -Service to get a list of pending jobs and -then execution operations on these pending jobs until there are no more -remaining on the device. Imagine periodic software updates that must be sent to and -executed on devices in the wild. - -This sample requires you to create jobs for your device to execute. See -[instructions here](https://docs.aws.amazon.com/iot/latest/developerguide/create-manage-jobs.html). - -On startup, the sample tries to get a list of all the in-progress and queued -jobs and display them in a list. Then it tries to start the next pending job execution. -If such a job exists, the sample emulates "doing work" by spawning a thread -that sleeps for several seconds before marking the job as SUCCEEDED. When no -pending job executions exist, the sample sits in an idle state. - -The sample also subscribes to receive "Next Job Execution Changed" events. -If the sample is idle, this event wakes it to start the job. If the sample is -already working on a job, it remembers to try for another when it's done. -This event is sent by the service when the current job completes, so the -sample will be continually prompted to try another job until none remain. - -Source: `samples/jobs.py` - -Run the sample like this: ``` sh -# For Windows: replace 'python3' with 'python' -python3 jobs.py --endpoint --ca_file --cert --key --thing_name +# For Windows: replace 'python3' with 'python' and '/' with '\' +python3 pubsub/pubsub.py --help ``` -Your Thing's [Policy](https://docs.aws.amazon.com/iot/latest/developerguide/iot-policies.html) must provide privileges for this sample to connect, subscribe, publish, and receive. Make sure your policy allows a client ID of `test-*` to connect or use `--client_id ` to send the client ID your policy supports. - -
-Sample Policy -
-{
-  "Version": "2012-10-17",
-  "Statement": [
-    {
-      "Effect": "Allow",
-      "Action": "iot:Publish",
-      "Resource": [
-        "arn:aws:iot:region:account:topic/$aws/things/thingname/jobs/start-next",
-        "arn:aws:iot:region:account:topic/$aws/things/thingname/jobs/*/update",
-        "arn:aws:iot:region:account:topic/$aws/things/thingname/jobs/*/get",
-        "arn:aws:iot:region:account:topic/$aws/things/thingname/jobs/get"
-      ]
-    },
-    {
-      "Effect": "Allow",
-      "Action": "iot:Receive",
-      "Resource": [
-        "arn:aws:iot:region:account:topic/$aws/things/thingname/jobs/notify-next",
-        "arn:aws:iot:region:account:topic/$aws/things/thingname/jobs/start-next/*",
-        "arn:aws:iot:region:account:topic/$aws/things/thingname/jobs/*/update/*",
-        "arn:aws:iot:region:account:topic/$aws/things/thingname/jobs/get/*",
-        "arn:aws:iot:region:account:topic/$aws/things/thingname/jobs/*/get/*"
-      ]
-    },
-    {
-      "Effect": "Allow",
-      "Action": "iot:Subscribe",
-      "Resource": [
-        "arn:aws:iot:region:account:topicfilter/$aws/things/thingname/jobs/notify-next",
-        "arn:aws:iot:region:account:topicfilter/$aws/things/thingname/jobs/start-next/*",
-        "arn:aws:iot:region:account:topicfilter/$aws/things/thingname/jobs/*/update/*",
-        "arn:aws:iot:region:account:topicfilter/$aws/things/thingname/jobs/get/*",
-        "arn:aws:iot:region:account:topicfilter/$aws/things/thingname/jobs/*/get/*"
-      ]
-    },
-    {
-      "Effect": "Allow",
-      "Action": "iot:Connect",
-      "Resource": "arn:aws:iot:region:account:client/test-*"
-    }
-  ]
-}
-
-
- -## Fleet Provisioning +Which will result in output showing all of the options that can be passed in at the command line, along with descriptions of what each does and whether they are optional or not. -This sample uses the AWS IoT -[Fleet provisioning](https://docs.aws.amazon.com/iot/latest/developerguide/provision-wo-cert.html) -to provision devices using either a CSR or Keys-And-Certificate and subsequently calls RegisterThing. +### Enable logging in samples -On startup, the script subscribes to topics based on the request type of either CSR or Keys topics, -publishes the request to corresponding topic and calls RegisterThing. +To enable logging in the samples, you need to pass the `--verbosity` as an additional argument. `--verbosity` controls the level of logging shown. `--verbosity` can be set to `Trace`, `Debug`, `Info`, `Warn`, `Error`, `Fatal`, or `None`. -Source: `samples/fleetprovisioning.py` +For example, to run [PubSub](./pubsub/README.md) sample with logging you could use the following: -Run the sample using createKeysAndCertificate: ``` sh -# For Windows: replace 'python3' with 'python' -python3 fleetprovisioning.py --endpoint --ca_file --cert --key --template_name --template_parameters +# For Windows: replace 'python3' with 'python' and '/' with '\' +python3 pubsub/pubsub.py --verbosity Debug ``` - -Run the sample using createCertificateFromCsr: -``` sh -# For Windows: replace 'python3' with 'python' -python3 fleetprovisioning.py --endpoint --ca_file --cert --key --template_name --template_parameters --csr -``` - -Your Thing's [Policy](https://docs.aws.amazon.com/iot/latest/developerguide/iot-policies.html) must provide privileges for this sample to connect, subscribe, publish, and receive. Make sure your policy allows a client ID of `test-*` to connect or use `--client_id ` to send the client ID your policy supports. - -
-(see sample policy) -
-{
-  "Version": "2012-10-17",
-  "Statement": [
-    {
-      "Effect": "Allow",
-      "Action": "iot:Publish",
-      "Resource": [
-        "arn:aws:iot:region:account:topic/$aws/certificates/create/json",
-        "arn:aws:iot:region:account:topic/$aws/certificates/create-from-csr/json",
-        "arn:aws:iot:region:account:topic/$aws/provisioning-templates/templatename/provision/json"
-      ]
-    },
-    {
-      "Effect": "Allow",
-      "Action": [
-        "iot:Receive"
-      ],
-      "Resource": [
-        "arn:aws:iot:region:account:topic/$aws/certificates/create/json/accepted",
-        "arn:aws:iot:region:account:topic/$aws/certificates/create/json/rejected",
-        "arn:aws:iot:region:account:topic/$aws/certificates/create-from-csr/json/accepted",
-        "arn:aws:iot:region:account:topic/$aws/certificates/create-from-csr/json/rejected",
-        "arn:aws:iot:region:account:topic/$aws/provisioning-templates/templatename/provision/json/accepted",
-        "arn:aws:iot:region:account:topic/$aws/provisioning-templates/templatename/provision/json/rejected"
-      ]
-    },
-    {
-      "Effect": "Allow",
-      "Action": [
-        "iot:Subscribe"
-      ],
-      "Resource": [
-        "arn:aws:iot:region:account:topicfilter/$aws/certificates/create/json/accepted",
-        "arn:aws:iot:region:account:topicfilter/$aws/certificates/create/json/rejected",
-        "arn:aws:iot:region:account:topicfilter/$aws/certificates/create-from-csr/json/accepted",
-        "arn:aws:iot:region:account:topicfilter/$aws/certificates/create-from-csr/json/rejected",
-        "arn:aws:iot:region:account:topicfilter/$aws/provisioning-templates/templatename/provision/json/accepted",
-        "arn:aws:iot:region:account:topicfilter/$aws/provisioning-templates/templatename/provision/json/rejected"
-      ]
-    },
-    {
-      "Effect": "Allow",
-      "Action": "iot:Connect",
-      "Resource": "arn:aws:iot:region:account:client/test-*"
-    }
-  ]
-}
-
-
- -### Fleet Provisioning Detailed Instructions - -#### AWS Resource Setup - -Fleet provisioning requires some additional AWS resources be set up first. This section documents the steps you need to take to -get the sample up and running. These steps assume you have the AWS CLI installed and the default user/credentials has -sufficient permission to perform all of the listed operations. These steps are based on provisioning setup steps -that can be found at [Embedded C SDK Setup](https://docs.aws.amazon.com/freertos/latest/lib-ref/c-sdk/provisioning/provisioning_tests.html#provisioning_system_tests_setup). - -First, create the IAM role that will be needed by the fleet provisioning template. Replace `RoleName` with a name of the role you want to create. -``` sh -aws iam create-role \ - --role-name [RoleName] \ - --assume-role-policy-document '{"Version":"2012-10-17","Statement":[{"Action":"sts:AssumeRole","Effect":"Allow","Principal":{"Service":"iot.amazonaws.com"}}]}' -``` -Next, attach a policy to the role created in the first step. Replace `RoleName` with the name of the role you created previously. -``` sh -aws iam attach-role-policy \ - --role-name [RoleName] \ - --policy-arn arn:aws:iam::aws:policy/service-role/AWSIoTThingsRegistration -``` -Finally, create the template resource which will be used for provisioning by the demo application. This needs to be done only -once. To create a template, the following AWS CLI command may be used. Replace `TemplateName` with the name of the fleet -provisioning template you want to create. Replace `RoleName` with the name of the role you created previously. Replace -`TemplateJSON` with the template body as a JSON string (containing escape characters). Replace `account` with your AWS -account number. -``` sh -aws iot create-provisioning-template \ - --template-name [TemplateName] \ - --provisioning-role-arn arn:aws:iam::[account]:role/[RoleName] \ - --template-body "[TemplateJSON]" \ - --enabled -``` -The rest of the instructions assume you have used the following for the template body: - -
-(see template body) -``` sh -{ - "Parameters": { - "DeviceLocation": { - "Type": "String" - }, - "AWS::IoT::Certificate::Id": { - "Type": "String" - }, - "SerialNumber": { - "Type": "String" - } - }, - "Mappings": { - "LocationTable": { - "Seattle": { - "LocationUrl": "https://example.aws" - } - } - }, - "Resources": { - "thing": { - "Type": "AWS::IoT::Thing", - "Properties": { - "ThingName": { - "Fn::Join": [ - "", - [ - "ThingPrefix_", - { - "Ref": "SerialNumber" - } - ] - ] - }, - "AttributePayload": { - "version": "v1", - "serialNumber": "serialNumber" - } - }, - "OverrideSettings": { - "AttributePayload": "MERGE", - "ThingTypeName": "REPLACE", - "ThingGroups": "DO_NOTHING" - } - }, - "certificate": { - "Type": "AWS::IoT::Certificate", - "Properties": { - "CertificateId": { - "Ref": "AWS::IoT::Certificate::Id" - }, - "Status": "Active" - }, - "OverrideSettings": { - "Status": "REPLACE" - } - }, - "policy": { - "Type": "AWS::IoT::Policy", - "Properties": { - "PolicyDocument": { - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Action": [ - "iot:Connect", - "iot:Subscribe", - "iot:Publish", - "iot:Receive" - ], - "Resource": "*" - } - ] - } - } - } - }, - "DeviceConfiguration": { - "FallbackUrl": "https://www.example.com/test-site", - "LocationUrl": { - "Fn::FindInMap": [ - "LocationTable", - { - "Ref": "DeviceLocation" - }, - "LocationUrl" - ] - } - } -} -``` -
- -If you use a different body, you may need to pass in different template parameters. - -#### Running the sample and provisioning using a certificate-key set from a provisioning claim - -To run the provisioning sample, you'll need a certificate and key set with sufficient permissions. Provisioning certificates are normally -created ahead of time and placed on your device, but for this sample, we will just create them on the fly. You can also -use any certificate set you've already created if it has sufficient IoT permissions and in doing so, you can skip the step -that calls `create-provisioning-claim`. - -We've included a script in the utils folder that creates certificate and key files from the response of calling -`create-provisioning-claim`. These dynamically sourced certificates are only valid for five minutes. When running the command, -you'll need to substitute the name of the template you previously created, and on Windows, replace the paths with something appropriate. - -(Optional) Create a temporary provisioning claim certificate set: -``` sh -aws iot create-provisioning-claim \ - --template-name [TemplateName] \ - | python3 ../utils/parse_cert_set_result.py \ - --path /tmp \ - --filename provision -``` - -The provisioning claim's cert and key set have been written to `/tmp/provision*`. Now you can use these temporary keys -to perform the actual provisioning. If you are not using the temporary provisioning certificate, replace the paths for `--cert` -and `--key` appropriately: - -``` sh -# For Windows: replace 'python3' with 'python' -python3 fleetprovisioning.py \ - --endpoint \ - --ca_file \ - --cert \ - --key \ - --template_name