From dfb5b0f05e07dae2773dbe2b8d5cfdbcbe2cbf53 Mon Sep 17 00:00:00 2001 From: Michael Brewer Date: Thu, 4 Mar 2021 01:02:17 -0800 Subject: [PATCH 01/11] refactor(idempotent): Create a config class --- .../utilities/idempotency/config.py | 38 +++++ .../utilities/idempotency/idempotency.py | 16 +- .../utilities/idempotency/persistence/base.py | 61 +++---- .../idempotency/persistence/dynamodb.py | 8 +- tests/functional/idempotency/conftest.py | 27 ++- .../idempotency/test_idempotency.py | 158 ++++++++---------- 6 files changed, 160 insertions(+), 148 deletions(-) create mode 100644 aws_lambda_powertools/utilities/idempotency/config.py diff --git a/aws_lambda_powertools/utilities/idempotency/config.py b/aws_lambda_powertools/utilities/idempotency/config.py new file mode 100644 index 00000000000..a6f742225e3 --- /dev/null +++ b/aws_lambda_powertools/utilities/idempotency/config.py @@ -0,0 +1,38 @@ +class IdempotencyConfig: + def __init__( + self, + event_key_jmespath: str = "", + payload_validation_jmespath: str = "", + raise_on_no_idempotency_key: bool = False, + expires_after_seconds: int = 60 * 60, # 1 hour default + use_local_cache: bool = False, + local_cache_max_items: int = 256, + hash_function: str = "md5", + ): + """ + Initialize the base persistence layer + + Parameters + ---------- + event_key_jmespath: str + A jmespath expression to extract the idempotency key from the event record + payload_validation_jmespath: str + A jmespath expression to extract the payload to be validated from the event record + raise_on_no_idempotency_key: bool, optional + Raise exception if no idempotency key was found in the request, by default False + expires_after_seconds: int + The number of seconds to wait before a record is expired + use_local_cache: bool, optional + Whether to locally cache idempotency results, by default False + local_cache_max_items: int, optional + Max number of items to store in local cache, by default 1024 + hash_function: str, optional + Function to use for calculating hashes, by default md5. + """ + self.event_key_jmespath = event_key_jmespath + self.payload_validation_jmespath = payload_validation_jmespath + self.raise_on_no_idempotency_key = raise_on_no_idempotency_key + self.expires_after_seconds = expires_after_seconds + self.use_local_cache = use_local_cache + self.local_cache_max_items = local_cache_max_items + self.hash_function = hash_function diff --git a/aws_lambda_powertools/utilities/idempotency/idempotency.py b/aws_lambda_powertools/utilities/idempotency/idempotency.py index bc556f49912..796729ede68 100644 --- a/aws_lambda_powertools/utilities/idempotency/idempotency.py +++ b/aws_lambda_powertools/utilities/idempotency/idempotency.py @@ -5,6 +5,7 @@ from typing import Any, Callable, Dict, Optional from aws_lambda_powertools.middleware_factory import lambda_handler_decorator +from aws_lambda_powertools.utilities.idempotency.config import IdempotencyConfig from aws_lambda_powertools.utilities.idempotency.exceptions import ( IdempotencyAlreadyInProgressError, IdempotencyInconsistentStateError, @@ -29,6 +30,7 @@ def idempotent( event: Dict[str, Any], context: LambdaContext, persistence_store: BasePersistenceLayer, + config: IdempotencyConfig = None, ) -> Any: """ Middleware to handle idempotency @@ -43,20 +45,24 @@ def idempotent( Lambda's Context persistence_store: BasePersistenceLayer Instance of BasePersistenceLayer to store data + config: IdempotencyConfig + Configutation Examples -------- **Processes Lambda's event in an idempotent manner** - >>> from aws_lambda_powertools.utilities.idempotency import idempotent, DynamoDBPersistenceLayer + >>> from aws_lambda_powertools.utilities.idempotency import ( + >>> idempotent, DynamoDBPersistenceLayer + >>> ) >>> - >>> persistence_store = DynamoDBPersistenceLayer(event_key_jmespath="body", table_name="idempotency_store") + >>> persistence_layer = DynamoDBPersistenceLayer(table_name="idempotency_store") >>> - >>> @idempotent(persistence_store=persistence_store) + >>> @idempotent(persistence_store=persistence_layer, config=IdempotencyConfig(event_key_jmespath="body")) >>> def handler(event, context): >>> return {"StatusCode": 200} """ - idempotency_handler = IdempotencyHandler(handler, event, context, persistence_store) + idempotency_handler = IdempotencyHandler(handler, event, context, config, persistence_store) # IdempotencyInconsistentStateError can happen under rare but expected cases when persistent state changes in the # small time between put & get requests. In most cases we can retry successfully on this exception. @@ -82,6 +88,7 @@ def __init__( lambda_handler: Callable[[Any, LambdaContext], Any], event: Dict[str, Any], context: LambdaContext, + config: IdempotencyConfig, persistence_store: BasePersistenceLayer, ): """ @@ -98,6 +105,7 @@ def __init__( persistence_store : BasePersistenceLayer Instance of persistence layer to store idempotency records """ + persistence_store.configure(config) self.persistence_store = persistence_store self.context = context self.event = event diff --git a/aws_lambda_powertools/utilities/idempotency/persistence/base.py b/aws_lambda_powertools/utilities/idempotency/persistence/base.py index c866d75d98d..3bd96f71430 100644 --- a/aws_lambda_powertools/utilities/idempotency/persistence/base.py +++ b/aws_lambda_powertools/utilities/idempotency/persistence/base.py @@ -21,6 +21,7 @@ IdempotencyKeyError, IdempotencyValidationError, ) +from aws_lambda_powertools.utilities.idempotency.idempotency import IdempotencyConfig logger = logging.getLogger(__name__) @@ -106,49 +107,39 @@ class BasePersistenceLayer(ABC): Abstract Base Class for Idempotency persistence layer. """ - def __init__( - self, - event_key_jmespath: str = "", - payload_validation_jmespath: str = "", - expires_after_seconds: int = 60 * 60, # 1 hour default - use_local_cache: bool = False, - local_cache_max_items: int = 256, - hash_function: str = "md5", - raise_on_no_idempotency_key: bool = False, - ) -> None: + def __init__(self): + self.event_key_jmespath = None + self.event_key_compiled_jmespath = None + self.payload_validation_enabled = None + self.validation_key_jmespath = None + self.raise_on_no_idempotency_key = None + self.expires_after_seconds = None + self.use_local_cache = None + self._cache = None + self.hash_function = None + + def configure(self, config: IdempotencyConfig,) -> None: """ Initialize the base persistence layer Parameters ---------- - event_key_jmespath: str - A jmespath expression to extract the idempotency key from the event record - payload_validation_jmespath: str - A jmespath expression to extract the payload to be validated from the event record - expires_after_seconds: int - The number of seconds to wait before a record is expired - use_local_cache: bool, optional - Whether to locally cache idempotency results, by default False - local_cache_max_items: int, optional - Max number of items to store in local cache, by default 1024 - hash_function: str, optional - Function to use for calculating hashes, by default md5. - raise_on_no_idempotency_key: bool, optional - Raise exception if no idempotency key was found in the request, by default False - """ - self.event_key_jmespath = event_key_jmespath + config: IdempotencyConfig + Configuration settings + """ + self.event_key_jmespath = config.event_key_jmespath if self.event_key_jmespath: - self.event_key_compiled_jmespath = jmespath.compile(event_key_jmespath) - self.expires_after_seconds = expires_after_seconds - self.use_local_cache = use_local_cache - if self.use_local_cache: - self._cache = LRUDict(max_items=local_cache_max_items) + self.event_key_compiled_jmespath = jmespath.compile(config.event_key_jmespath) self.payload_validation_enabled = False - if payload_validation_jmespath: - self.validation_key_jmespath = jmespath.compile(payload_validation_jmespath) + if config.payload_validation_jmespath: + self.validation_key_jmespath = jmespath.compile(config.payload_validation_jmespath) self.payload_validation_enabled = True - self.hash_function = getattr(hashlib, hash_function) - self.raise_on_no_idempotency_key = raise_on_no_idempotency_key + self.raise_on_no_idempotency_key = config.raise_on_no_idempotency_key + self.expires_after_seconds = config.expires_after_seconds + self.use_local_cache = config.use_local_cache + if self.use_local_cache: + self._cache = LRUDict(max_items=config.local_cache_max_items) + self.hash_function = getattr(hashlib, config.hash_function) def _get_hashed_idempotency_key(self, lambda_event: Dict[str, Any]) -> str: """ diff --git a/aws_lambda_powertools/utilities/idempotency/persistence/dynamodb.py b/aws_lambda_powertools/utilities/idempotency/persistence/dynamodb.py index 4d66448755d..d87cd71ff4e 100644 --- a/aws_lambda_powertools/utilities/idempotency/persistence/dynamodb.py +++ b/aws_lambda_powertools/utilities/idempotency/persistence/dynamodb.py @@ -26,8 +26,6 @@ def __init__( validation_key_attr: str = "validation", boto_config: Optional[Config] = None, boto3_session: Optional[boto3.session.Session] = None, - *args, - **kwargs, ): """ Initialize the DynamoDB client @@ -57,9 +55,9 @@ def __init__( **Create a DynamoDB persistence layer with custom settings** >>> from aws_lambda_powertools.utilities.idempotency import idempotent, DynamoDBPersistenceLayer >>> - >>> persistence_store = DynamoDBPersistenceLayer(event_key="body", table_name="idempotency_store") + >>> persistence_store = DynamoDBPersistenceLayer(table_name="idempotency_store") >>> - >>> @idempotent(persistence_store=persistence_store) + >>> @idempotent(persistence_store=persistence_store, event_key="body") >>> def handler(event, context): >>> return {"StatusCode": 200} """ @@ -74,7 +72,7 @@ def __init__( self.status_attr = status_attr self.data_attr = data_attr self.validation_key_attr = validation_key_attr - super(DynamoDBPersistenceLayer, self).__init__(*args, **kwargs) + super(DynamoDBPersistenceLayer, self).__init__() def _item_to_data_record(self, item: Dict[str, Any]) -> DataRecord: """ diff --git a/tests/functional/idempotency/conftest.py b/tests/functional/idempotency/conftest.py index 68492648337..d3928b4db41 100644 --- a/tests/functional/idempotency/conftest.py +++ b/tests/functional/idempotency/conftest.py @@ -12,6 +12,7 @@ from aws_lambda_powertools.shared.json_encoder import Encoder from aws_lambda_powertools.utilities.idempotency import DynamoDBPersistenceLayer +from aws_lambda_powertools.utilities.idempotency.idempotency import IdempotencyConfig from aws_lambda_powertools.utilities.validation import envelopes from aws_lambda_powertools.utilities.validation.base import unwrap_event_from_envelope @@ -150,34 +151,30 @@ def hashed_validation_key(lambda_apigw_event): @pytest.fixture -def persistence_store(config, request, default_jmespath): - persistence_store = DynamoDBPersistenceLayer( +def persistence_store(config): + return DynamoDBPersistenceLayer(table_name=TABLE_NAME, boto_config=config,) + + +@pytest.fixture +def idempotency_config(config, request, default_jmespath): + return IdempotencyConfig( event_key_jmespath=request.param.get("event_key_jmespath") or default_jmespath, - table_name=TABLE_NAME, - boto_config=config, use_local_cache=request.param["use_local_cache"], ) - return persistence_store @pytest.fixture -def persistence_store_without_jmespath(config, request): - persistence_store = DynamoDBPersistenceLayer( - table_name=TABLE_NAME, boto_config=config, use_local_cache=request.param["use_local_cache"], - ) - return persistence_store +def config_without_jmespath(config, request): + return IdempotencyConfig(use_local_cache=request.param["use_local_cache"],) @pytest.fixture -def persistence_store_with_validation(config, request, default_jmespath): - persistence_store = DynamoDBPersistenceLayer( +def config_with_validation(config, request, default_jmespath): + return IdempotencyConfig( event_key_jmespath=default_jmespath, - table_name=TABLE_NAME, - boto_config=config, use_local_cache=request.param, payload_validation_jmespath="requestContext", ) - return persistence_store @pytest.fixture diff --git a/tests/functional/idempotency/test_idempotency.py b/tests/functional/idempotency/test_idempotency.py index 675d8d1c5c8..46eb156c704 100644 --- a/tests/functional/idempotency/test_idempotency.py +++ b/tests/functional/idempotency/test_idempotency.py @@ -23,8 +23,9 @@ # Using parametrize to run test twice, with two separate instances of persistence store. One instance with caching # enabled, and one without. -@pytest.mark.parametrize("persistence_store", [{"use_local_cache": False}, {"use_local_cache": True}], indirect=True) +@pytest.mark.parametrize("idempotency_config", [{"use_local_cache": False}, {"use_local_cache": True}], indirect=True) def test_idempotent_lambda_already_completed( + idempotency_config, persistence_store, lambda_apigw_event, timestamp_future, @@ -55,7 +56,7 @@ def test_idempotent_lambda_already_completed( stubber.add_response("get_item", ddb_response, expected_params) stubber.activate() - @idempotent(persistence_store=persistence_store) + @idempotent(persistence_store=persistence_store, config=idempotency_config) def lambda_handler(event, context): raise Exception @@ -66,9 +67,9 @@ def lambda_handler(event, context): stubber.deactivate() -@pytest.mark.parametrize("persistence_store", [{"use_local_cache": False}, {"use_local_cache": True}], indirect=True) +@pytest.mark.parametrize("idempotency_config", [{"use_local_cache": False}, {"use_local_cache": True}], indirect=True) def test_idempotent_lambda_in_progress( - persistence_store, lambda_apigw_event, lambda_response, timestamp_future, hashed_idempotency_key + idempotency_config, persistence_store, lambda_apigw_event, lambda_response, timestamp_future, hashed_idempotency_key ): """ Test idempotent decorator where lambda_handler is already processing an event with matching event key @@ -93,7 +94,7 @@ def test_idempotent_lambda_in_progress( stubber.add_response("get_item", ddb_response, expected_params) stubber.activate() - @idempotent(persistence_store=persistence_store) + @idempotent(persistence_store=persistence_store, config=idempotency_config) def lambda_handler(event, context): return lambda_response @@ -109,9 +110,15 @@ def lambda_handler(event, context): @pytest.mark.skipif(sys.version_info < (3, 8), reason="issue with pytest mock lib for < 3.8") -@pytest.mark.parametrize("persistence_store", [{"use_local_cache": True}], indirect=True) +@pytest.mark.parametrize("idempotency_config", [{"use_local_cache": True}], indirect=True) def test_idempotent_lambda_in_progress_with_cache( - persistence_store, lambda_apigw_event, lambda_response, timestamp_future, hashed_idempotency_key, mocker + idempotency_config, + persistence_store, + lambda_apigw_event, + lambda_response, + timestamp_future, + hashed_idempotency_key, + mocker, ): """ Test idempotent decorator where lambda_handler is already processing an event with matching event key, cache @@ -144,7 +151,7 @@ def test_idempotent_lambda_in_progress_with_cache( stubber.add_response("get_item", copy.deepcopy(ddb_response), copy.deepcopy(expected_params)) stubber.activate() - @idempotent(persistence_store=persistence_store) + @idempotent(persistence_store=persistence_store, config=idempotency_config) def lambda_handler(event, context): return lambda_response @@ -167,8 +174,9 @@ def lambda_handler(event, context): stubber.deactivate() -@pytest.mark.parametrize("persistence_store", [{"use_local_cache": False}, {"use_local_cache": True}], indirect=True) +@pytest.mark.parametrize("idempotency_config", [{"use_local_cache": False}, {"use_local_cache": True}], indirect=True) def test_idempotent_lambda_first_execution( + idempotency_config, persistence_store, lambda_apigw_event, expected_params_update_item, @@ -189,63 +197,19 @@ def test_idempotent_lambda_first_execution( stubber.add_response("update_item", ddb_response, expected_params_update_item) stubber.activate() - @idempotent(persistence_store=persistence_store) - def lambda_handler(event, context): - return lambda_response - - lambda_handler(lambda_apigw_event, {}) - - stubber.assert_no_pending_responses() - stubber.deactivate() - - -@pytest.mark.skipif(sys.version_info < (3, 8), reason="issue with pytest mock lib for < 3.8") -@pytest.mark.parametrize("persistence_store", [{"use_local_cache": True}], indirect=True) -def test_idempotent_lambda_first_execution_cached( - persistence_store, - lambda_apigw_event, - expected_params_update_item, - expected_params_put_item, - lambda_response, - hashed_idempotency_key, - mocker, -): - """ - Test idempotent decorator when lambda is executed with an event with a previously unknown event key. Ensure - result is cached locally on the persistence store instance. - """ - save_to_cache_spy = mocker.spy(persistence_store, "_save_to_cache") - retrieve_from_cache_spy = mocker.spy(persistence_store, "_retrieve_from_cache") - stubber = stub.Stubber(persistence_store.table.meta.client) - ddb_response = {} - - stubber.add_response("put_item", ddb_response, expected_params_put_item) - stubber.add_response("update_item", ddb_response, expected_params_update_item) - stubber.activate() - - @idempotent(persistence_store=persistence_store) + @idempotent(persistence_store=persistence_store, config=idempotency_config) def lambda_handler(event, context): return lambda_response lambda_handler(lambda_apigw_event, {}) - retrieve_from_cache_spy.assert_called_once() - save_to_cache_spy.assert_called_once() - assert save_to_cache_spy.call_args[0][0].status == "COMPLETED" - assert persistence_store._cache.get(hashed_idempotency_key).status == "COMPLETED" - - # This lambda call should not call AWS API - lambda_handler(lambda_apigw_event, {}) - assert retrieve_from_cache_spy.call_count == 3 - retrieve_from_cache_spy.assert_called_with(idempotency_key=hashed_idempotency_key) - - # This assertion fails if an AWS API operation was called more than once stubber.assert_no_pending_responses() stubber.deactivate() -@pytest.mark.parametrize("persistence_store", [{"use_local_cache": False}, {"use_local_cache": True}], indirect=True) +@pytest.mark.parametrize("idempotency_config", [{"use_local_cache": False}, {"use_local_cache": True}], indirect=True) def test_idempotent_lambda_expired( + idempotency_config, persistence_store, lambda_apigw_event, timestamp_expired, @@ -267,7 +231,7 @@ def test_idempotent_lambda_expired( stubber.add_response("update_item", ddb_response, expected_params_update_item) stubber.activate() - @idempotent(persistence_store=persistence_store) + @idempotent(persistence_store=persistence_store, config=idempotency_config) def lambda_handler(event, context): return lambda_response @@ -277,8 +241,9 @@ def lambda_handler(event, context): stubber.deactivate() -@pytest.mark.parametrize("persistence_store", [{"use_local_cache": False}, {"use_local_cache": True}], indirect=True) +@pytest.mark.parametrize("idempotency_config", [{"use_local_cache": False}, {"use_local_cache": True}], indirect=True) def test_idempotent_lambda_exception( + idempotency_config, persistence_store, lambda_apigw_event, timestamp_future, @@ -303,7 +268,7 @@ def test_idempotent_lambda_exception( stubber.add_response("delete_item", ddb_response, expected_params_delete_item) stubber.activate() - @idempotent(persistence_store=persistence_store) + @idempotent(persistence_store=persistence_store, config=idempotency_config) def lambda_handler(event, context): raise Exception("Something went wrong!") @@ -315,10 +280,11 @@ def lambda_handler(event, context): @pytest.mark.parametrize( - "persistence_store_with_validation", [{"use_local_cache": False}, {"use_local_cache": True}], indirect=True + "config_with_validation", [{"use_local_cache": False}, {"use_local_cache": True}], indirect=True ) def test_idempotent_lambda_already_completed_with_validation_bad_payload( - persistence_store_with_validation, + config_with_validation, + persistence_store, lambda_apigw_event, timestamp_future, lambda_response, @@ -329,7 +295,7 @@ def test_idempotent_lambda_already_completed_with_validation_bad_payload( Test idempotent decorator where event with matching event key has already been succesfully processed """ - stubber = stub.Stubber(persistence_store_with_validation.table.meta.client) + stubber = stub.Stubber(persistence_store.table.meta.client) ddb_response = { "Item": { "id": {"S": hashed_idempotency_key}, @@ -346,7 +312,7 @@ def test_idempotent_lambda_already_completed_with_validation_bad_payload( stubber.add_response("get_item", ddb_response, expected_params) stubber.activate() - @idempotent(persistence_store=persistence_store_with_validation) + @idempotent(persistence_store=persistence_store, config=config_with_validation) def lambda_handler(event, context): return lambda_response @@ -358,8 +324,9 @@ def lambda_handler(event, context): stubber.deactivate() -@pytest.mark.parametrize("persistence_store", [{"use_local_cache": False}, {"use_local_cache": True}], indirect=True) +@pytest.mark.parametrize("idempotency_config", [{"use_local_cache": False}, {"use_local_cache": True}], indirect=True) def test_idempotent_lambda_expired_during_request( + idempotency_config, persistence_store, lambda_apigw_event, timestamp_expired, @@ -401,7 +368,7 @@ def test_idempotent_lambda_expired_during_request( stubber.activate() - @idempotent(persistence_store=persistence_store) + @idempotent(persistence_store=persistence_store, config=idempotency_config) def lambda_handler(event, context): return lambda_response @@ -413,8 +380,9 @@ def lambda_handler(event, context): stubber.deactivate() -@pytest.mark.parametrize("persistence_store", [{"use_local_cache": False}, {"use_local_cache": True}], indirect=True) +@pytest.mark.parametrize("idempotency_config", [{"use_local_cache": False}, {"use_local_cache": True}], indirect=True) def test_idempotent_persistence_exception_deleting( + idempotency_config, persistence_store, lambda_apigw_event, timestamp_future, @@ -434,7 +402,7 @@ def test_idempotent_persistence_exception_deleting( stubber.add_client_error("delete_item", "UnrecoverableError") stubber.activate() - @idempotent(persistence_store=persistence_store) + @idempotent(persistence_store=persistence_store, config=idempotency_config) def lambda_handler(event, context): raise Exception("Something went wrong!") @@ -446,8 +414,9 @@ def lambda_handler(event, context): stubber.deactivate() -@pytest.mark.parametrize("persistence_store", [{"use_local_cache": False}, {"use_local_cache": True}], indirect=True) +@pytest.mark.parametrize("idempotency_config", [{"use_local_cache": False}, {"use_local_cache": True}], indirect=True) def test_idempotent_persistence_exception_updating( + idempotency_config, persistence_store, lambda_apigw_event, timestamp_future, @@ -467,7 +436,7 @@ def test_idempotent_persistence_exception_updating( stubber.add_client_error("update_item", "UnrecoverableError") stubber.activate() - @idempotent(persistence_store=persistence_store) + @idempotent(persistence_store=persistence_store, config=idempotency_config) def lambda_handler(event, context): return {"message": "success!"} @@ -479,8 +448,9 @@ def lambda_handler(event, context): stubber.deactivate() -@pytest.mark.parametrize("persistence_store", [{"use_local_cache": False}, {"use_local_cache": True}], indirect=True) +@pytest.mark.parametrize("idempotency_config", [{"use_local_cache": False}, {"use_local_cache": True}], indirect=True) def test_idempotent_persistence_exception_getting( + idempotency_config, persistence_store, lambda_apigw_event, timestamp_future, @@ -498,7 +468,7 @@ def test_idempotent_persistence_exception_getting( stubber.add_client_error("get_item", "UnexpectedException") stubber.activate() - @idempotent(persistence_store=persistence_store) + @idempotent(persistence_store=persistence_store, config=idempotency_config) def lambda_handler(event, context): return {"message": "success!"} @@ -511,10 +481,11 @@ def lambda_handler(event, context): @pytest.mark.parametrize( - "persistence_store_with_validation", [{"use_local_cache": False}, {"use_local_cache": True}], indirect=True + "config_with_validation", [{"use_local_cache": False}, {"use_local_cache": True}], indirect=True ) def test_idempotent_lambda_first_execution_with_validation( - persistence_store_with_validation, + config_with_validation, + persistence_store, lambda_apigw_event, expected_params_update_item_with_validation, expected_params_put_item_with_validation, @@ -525,14 +496,14 @@ def test_idempotent_lambda_first_execution_with_validation( """ Test idempotent decorator when lambda is executed with an event with a previously unknown event key """ - stubber = stub.Stubber(persistence_store_with_validation.table.meta.client) + stubber = stub.Stubber(persistence_store.table.meta.client) ddb_response = {} stubber.add_response("put_item", ddb_response, expected_params_put_item_with_validation) stubber.add_response("update_item", ddb_response, expected_params_update_item_with_validation) stubber.activate() - @idempotent(persistence_store=persistence_store_with_validation) + @idempotent(persistence_store=persistence_store, config=config_with_validation) def lambda_handler(lambda_apigw_event, context): return lambda_response @@ -543,10 +514,11 @@ def lambda_handler(lambda_apigw_event, context): @pytest.mark.parametrize( - "persistence_store_without_jmespath", [{"use_local_cache": False}, {"use_local_cache": True}], indirect=True + "config_without_jmespath", [{"use_local_cache": False}, {"use_local_cache": True}], indirect=True ) def test_idempotent_lambda_with_validator_util( - persistence_store_without_jmespath, + config_without_jmespath, + persistence_store, lambda_apigw_event, timestamp_future, serialized_lambda_response, @@ -559,7 +531,7 @@ def test_idempotent_lambda_with_validator_util( validator utility to unwrap the event """ - stubber = stub.Stubber(persistence_store_without_jmespath.table.meta.client) + stubber = stub.Stubber(persistence_store.table.meta.client) ddb_response = { "Item": { "id": {"S": hashed_idempotency_key_with_envelope}, @@ -579,7 +551,7 @@ def test_idempotent_lambda_with_validator_util( stubber.activate() @validator(envelope=envelopes.API_GATEWAY_HTTP) - @idempotent(persistence_store=persistence_store_without_jmespath) + @idempotent(persistence_store=persistence_store, config=config_without_jmespath) def lambda_handler(event, context): mock_function() return "shouldn't get here!" @@ -600,10 +572,11 @@ def test_data_record_invalid_status_value(): assert e.value.args[0] == "UNSUPPORTED_STATUS" -@pytest.mark.parametrize("persistence_store", [{"use_local_cache": True}], indirect=True) -def test_in_progress_never_saved_to_cache(persistence_store): +@pytest.mark.parametrize("idempotency_config", [{"use_local_cache": True}], indirect=True) +def test_in_progress_never_saved_to_cache(idempotency_config, persistence_store): # GIVEN a data record with status "INPROGRESS" # and persistence_store has use_local_cache = True + persistence_store.configure(idempotency_config) data_record = DataRecord("key", status="INPROGRESS") # WHEN saving to local cache @@ -613,9 +586,10 @@ def test_in_progress_never_saved_to_cache(persistence_store): assert persistence_store._cache.get("key") is None -@pytest.mark.parametrize("persistence_store", [{"use_local_cache": False}], indirect=True) -def test_user_local_disabled(persistence_store): +@pytest.mark.parametrize("idempotency_config", [{"use_local_cache": False}], indirect=True) +def test_user_local_disabled(idempotency_config, persistence_store): # GIVEN a persistence_store with use_local_cache = False + persistence_store.configure(idempotency_config) # WHEN calling any local cache options data_record = DataRecord("key", status="COMPLETED") @@ -632,9 +606,11 @@ def test_user_local_disabled(persistence_store): assert not hasattr("persistence_store", "_cache") -@pytest.mark.parametrize("persistence_store", [{"use_local_cache": True}], indirect=True) -def test_delete_from_cache_when_empty(persistence_store): +@pytest.mark.parametrize("idempotency_config", [{"use_local_cache": True}], indirect=True) +def test_delete_from_cache_when_empty(idempotency_config, persistence_store): # GIVEN use_local_cache is True AND the local cache is empty + persistence_store.configure(idempotency_config) + try: # WHEN we _delete_from_cache persistence_store._delete_from_cache("key_does_not_exist") @@ -662,9 +638,12 @@ def test_is_missing_idempotency_key(): assert BasePersistenceLayer.is_missing_idempotency_key("") -@pytest.mark.parametrize("persistence_store", [{"use_local_cache": False, "event_key_jmespath": "body"}], indirect=True) -def test_default_no_raise_on_missing_idempotency_key(persistence_store): +@pytest.mark.parametrize( + "idempotency_config", [{"use_local_cache": False, "event_key_jmespath": "body"}], indirect=True +) +def test_default_no_raise_on_missing_idempotency_key(idempotency_config, persistence_store): # GIVEN a persistence_store with use_local_cache = False and event_key_jmespath = "body" + persistence_store.configure(idempotency_config) assert persistence_store.use_local_cache is False assert "body" in persistence_store.event_key_jmespath @@ -676,10 +655,11 @@ def test_default_no_raise_on_missing_idempotency_key(persistence_store): @pytest.mark.parametrize( - "persistence_store", [{"use_local_cache": False, "event_key_jmespath": "[body, x]"}], indirect=True + "idempotency_config", [{"use_local_cache": False, "event_key_jmespath": "[body, x]"}], indirect=True ) -def test_raise_on_no_idempotency_key(persistence_store): +def test_raise_on_no_idempotency_key(idempotency_config, persistence_store): # GIVEN a persistence_store with raise_on_no_idempotency_key and no idempotency key in the request + persistence_store.configure(idempotency_config) persistence_store.raise_on_no_idempotency_key = True assert persistence_store.use_local_cache is False assert "body" in persistence_store.event_key_jmespath From b4821523b63efa5039ce6901d49551503066fbcc Mon Sep 17 00:00:00 2001 From: Michael Brewer Date: Thu, 4 Mar 2021 01:26:14 -0800 Subject: [PATCH 02/11] fix: Reanable test --- .../utilities/idempotency/__init__.py | 4 +- .../utilities/idempotency/persistence/base.py | 23 +++++---- tests/functional/idempotency/conftest.py | 2 +- .../idempotency/test_idempotency.py | 47 +++++++++++++++++++ 4 files changed, 64 insertions(+), 12 deletions(-) diff --git a/aws_lambda_powertools/utilities/idempotency/__init__.py b/aws_lambda_powertools/utilities/idempotency/__init__.py index 98e2be15415..b46d0855a93 100644 --- a/aws_lambda_powertools/utilities/idempotency/__init__.py +++ b/aws_lambda_powertools/utilities/idempotency/__init__.py @@ -5,6 +5,6 @@ from aws_lambda_powertools.utilities.idempotency.persistence.base import BasePersistenceLayer from aws_lambda_powertools.utilities.idempotency.persistence.dynamodb import DynamoDBPersistenceLayer -from .idempotency import idempotent +from .idempotency import IdempotencyConfig, idempotent -__all__ = ("DynamoDBPersistenceLayer", "BasePersistenceLayer", "idempotent") +__all__ = ("DynamoDBPersistenceLayer", "BasePersistenceLayer", "idempotent", "IdempotencyConfig") diff --git a/aws_lambda_powertools/utilities/idempotency/persistence/base.py b/aws_lambda_powertools/utilities/idempotency/persistence/base.py index 3bd96f71430..7221e53ca13 100644 --- a/aws_lambda_powertools/utilities/idempotency/persistence/base.py +++ b/aws_lambda_powertools/utilities/idempotency/persistence/base.py @@ -9,19 +9,19 @@ import warnings from abc import ABC, abstractmethod from types import MappingProxyType -from typing import Any, Dict +from typing import Any, Dict, Optional import jmespath from aws_lambda_powertools.shared.cache_dict import LRUDict from aws_lambda_powertools.shared.json_encoder import Encoder +from aws_lambda_powertools.utilities.idempotency.config import IdempotencyConfig from aws_lambda_powertools.utilities.idempotency.exceptions import ( IdempotencyInvalidStatusError, IdempotencyItemAlreadyExistsError, IdempotencyKeyError, IdempotencyValidationError, ) -from aws_lambda_powertools.utilities.idempotency.idempotency import IdempotencyConfig logger = logging.getLogger(__name__) @@ -108,14 +108,16 @@ class BasePersistenceLayer(ABC): """ def __init__(self): - self.event_key_jmespath = None + self.configured = False + + self.event_key_jmespath: Optional[str] = None self.event_key_compiled_jmespath = None - self.payload_validation_enabled = None + self.payload_validation_enabled = False self.validation_key_jmespath = None - self.raise_on_no_idempotency_key = None + self.raise_on_no_idempotency_key = False self.expires_after_seconds = None - self.use_local_cache = None - self._cache = None + self.use_local_cache = False + self._cache: Optional[LRUDict] = None self.hash_function = None def configure(self, config: IdempotencyConfig,) -> None: @@ -127,10 +129,13 @@ def configure(self, config: IdempotencyConfig,) -> None: config: IdempotencyConfig Configuration settings """ + if self.configured: + # Temp hack to prevent being reconfigured. + return + self.configured = True self.event_key_jmespath = config.event_key_jmespath - if self.event_key_jmespath: + if config.event_key_jmespath: self.event_key_compiled_jmespath = jmespath.compile(config.event_key_jmespath) - self.payload_validation_enabled = False if config.payload_validation_jmespath: self.validation_key_jmespath = jmespath.compile(config.payload_validation_jmespath) self.payload_validation_enabled = True diff --git a/tests/functional/idempotency/conftest.py b/tests/functional/idempotency/conftest.py index d3928b4db41..60cbdac02ad 100644 --- a/tests/functional/idempotency/conftest.py +++ b/tests/functional/idempotency/conftest.py @@ -152,7 +152,7 @@ def hashed_validation_key(lambda_apigw_event): @pytest.fixture def persistence_store(config): - return DynamoDBPersistenceLayer(table_name=TABLE_NAME, boto_config=config,) + return DynamoDBPersistenceLayer(table_name=TABLE_NAME, boto_config=config) @pytest.fixture diff --git a/tests/functional/idempotency/test_idempotency.py b/tests/functional/idempotency/test_idempotency.py index 46eb156c704..845f8d6a237 100644 --- a/tests/functional/idempotency/test_idempotency.py +++ b/tests/functional/idempotency/test_idempotency.py @@ -207,6 +207,53 @@ def lambda_handler(event, context): stubber.deactivate() +@pytest.mark.skipif(sys.version_info < (3, 8), reason="issue with pytest mock lib for < 3.8") +@pytest.mark.parametrize("idempotency_config", [{"use_local_cache": True}], indirect=True) +def test_idempotent_lambda_first_execution_cached( + idempotency_config, + persistence_store, + lambda_apigw_event, + expected_params_update_item, + expected_params_put_item, + lambda_response, + hashed_idempotency_key, + mocker, +): + """ + Test idempotent decorator when lambda is executed with an event with a previously unknown event key. Ensure + result is cached locally on the persistence store instance. + """ + persistence_store.configure(idempotency_config) + save_to_cache_spy = mocker.spy(persistence_store, "_save_to_cache") + retrieve_from_cache_spy = mocker.spy(persistence_store, "_retrieve_from_cache") + stubber = stub.Stubber(persistence_store.table.meta.client) + ddb_response = {} + + stubber.add_response("put_item", ddb_response, expected_params_put_item) + stubber.add_response("update_item", ddb_response, expected_params_update_item) + stubber.activate() + + @idempotent(persistence_store=persistence_store, config=idempotency_config) + def lambda_handler(event, context): + return lambda_response + + lambda_handler(lambda_apigw_event, {}) + + retrieve_from_cache_spy.assert_called_once() + save_to_cache_spy.assert_called_once() + assert save_to_cache_spy.call_args[0][0].status == "COMPLETED" + assert persistence_store._cache.get(hashed_idempotency_key).status == "COMPLETED" + + # This lambda call should not call AWS API + lambda_handler(lambda_apigw_event, {}) + assert retrieve_from_cache_spy.call_count == 3 + retrieve_from_cache_spy.assert_called_with(idempotency_key=hashed_idempotency_key) + + # This assertion fails if an AWS API operation was called more than once + stubber.assert_no_pending_responses() + stubber.deactivate() + + @pytest.mark.parametrize("idempotency_config", [{"use_local_cache": False}, {"use_local_cache": True}], indirect=True) def test_idempotent_lambda_expired( idempotency_config, From 2512cda5a7189ec30188a4b6c460ab4b6fdb8c59 Mon Sep 17 00:00:00 2001 From: Michael Brewer Date: Thu, 4 Mar 2021 07:48:04 -0800 Subject: [PATCH 03/11] chore: Some refactoring --- .../utilities/idempotency/idempotency.py | 6 +++--- .../utilities/idempotency/persistence/base.py | 15 ++++++++------- examples/__init__.py | 0 tests/functional/idempotency/test_idempotency.py | 16 ++++++++-------- 4 files changed, 19 insertions(+), 18 deletions(-) create mode 100644 examples/__init__.py diff --git a/aws_lambda_powertools/utilities/idempotency/idempotency.py b/aws_lambda_powertools/utilities/idempotency/idempotency.py index 796729ede68..9ad0bdfdff3 100644 --- a/aws_lambda_powertools/utilities/idempotency/idempotency.py +++ b/aws_lambda_powertools/utilities/idempotency/idempotency.py @@ -52,7 +52,7 @@ def idempotent( -------- **Processes Lambda's event in an idempotent manner** >>> from aws_lambda_powertools.utilities.idempotency import ( - >>> idempotent, DynamoDBPersistenceLayer + >>> idempotent, DynamoDBPersistenceLayer, IdempotencyConfig >>> ) >>> >>> persistence_layer = DynamoDBPersistenceLayer(table_name="idempotency_store") @@ -62,7 +62,7 @@ def idempotent( >>> return {"StatusCode": 200} """ - idempotency_handler = IdempotencyHandler(handler, event, context, config, persistence_store) + idempotency_handler = IdempotencyHandler(handler, event, context, config or IdempotencyConfig(), persistence_store) # IdempotencyInconsistentStateError can happen under rare but expected cases when persistent state changes in the # small time between put & get requests. In most cases we can retry successfully on this exception. @@ -105,7 +105,7 @@ def __init__( persistence_store : BasePersistenceLayer Instance of persistence layer to store idempotency records """ - persistence_store.configure(config) + persistence_store._configure(config) self.persistence_store = persistence_store self.context = context self.event = event diff --git a/aws_lambda_powertools/utilities/idempotency/persistence/base.py b/aws_lambda_powertools/utilities/idempotency/persistence/base.py index 9a024fe9a0b..444b6bcc070 100644 --- a/aws_lambda_powertools/utilities/idempotency/persistence/base.py +++ b/aws_lambda_powertools/utilities/idempotency/persistence/base.py @@ -109,32 +109,33 @@ class BasePersistenceLayer(ABC): """ def __init__(self): + """Initialize the defaults """ self.configured = False - self.event_key_jmespath: Optional[str] = None self.event_key_compiled_jmespath = None self.jmespath_options: Optional[dict] = None self.payload_validation_enabled = False self.validation_key_jmespath = None self.raise_on_no_idempotency_key = False - self.expires_after_seconds = None + self.expires_after_seconds: int = 60 * 60 # 1 hour default self.use_local_cache = False self._cache: Optional[LRUDict] = None self.hash_function = None - def configure(self, config: IdempotencyConfig,) -> None: + def _configure(self, config: IdempotencyConfig) -> None: """ - Initialize the base persistence layer + Initialize the base persistence layer from the configuration settings Parameters ---------- config: IdempotencyConfig - Configuration settings + Idempotency configuration settings """ if self.configured: - # Temp hack to prevent being reconfigured. + # Prevent being reconfigured. return self.configured = True + self.event_key_jmespath = config.event_key_jmespath if config.event_key_jmespath: self.event_key_compiled_jmespath = jmespath.compile(config.event_key_jmespath) @@ -174,9 +175,9 @@ def _get_hashed_idempotency_key(self, lambda_event: Dict[str, Any]) -> str: ) if self.is_missing_idempotency_key(data): - warnings.warn(f"No value found for idempotency_key. jmespath: {self.event_key_jmespath}") if self.raise_on_no_idempotency_key: raise IdempotencyKeyError("No data found to create a hashed idempotency_key") + warnings.warn(f"No value found for idempotency_key. jmespath: {self.event_key_jmespath}") return self._generate_hash(data) diff --git a/examples/__init__.py b/examples/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/functional/idempotency/test_idempotency.py b/tests/functional/idempotency/test_idempotency.py index d6f43dafd12..ccebf7c1ba1 100644 --- a/tests/functional/idempotency/test_idempotency.py +++ b/tests/functional/idempotency/test_idempotency.py @@ -225,7 +225,7 @@ def test_idempotent_lambda_first_execution_cached( Test idempotent decorator when lambda is executed with an event with a previously unknown event key. Ensure result is cached locally on the persistence store instance. """ - persistence_store.configure(idempotency_config) + persistence_store._configure(idempotency_config) save_to_cache_spy = mocker.spy(persistence_store, "_save_to_cache") retrieve_from_cache_spy = mocker.spy(persistence_store, "_retrieve_from_cache") stubber = stub.Stubber(persistence_store.table.meta.client) @@ -625,7 +625,7 @@ def test_data_record_invalid_status_value(): def test_in_progress_never_saved_to_cache(idempotency_config, persistence_store): # GIVEN a data record with status "INPROGRESS" # and persistence_store has use_local_cache = True - persistence_store.configure(idempotency_config) + persistence_store._configure(idempotency_config) data_record = DataRecord("key", status="INPROGRESS") # WHEN saving to local cache @@ -638,7 +638,7 @@ def test_in_progress_never_saved_to_cache(idempotency_config, persistence_store) @pytest.mark.parametrize("idempotency_config", [{"use_local_cache": False}], indirect=True) def test_user_local_disabled(idempotency_config, persistence_store): # GIVEN a persistence_store with use_local_cache = False - persistence_store.configure(idempotency_config) + persistence_store._configure(idempotency_config) # WHEN calling any local cache options data_record = DataRecord("key", status="COMPLETED") @@ -658,7 +658,7 @@ def test_user_local_disabled(idempotency_config, persistence_store): @pytest.mark.parametrize("idempotency_config", [{"use_local_cache": True}], indirect=True) def test_delete_from_cache_when_empty(idempotency_config, persistence_store): # GIVEN use_local_cache is True AND the local cache is empty - persistence_store.configure(idempotency_config) + persistence_store._configure(idempotency_config) try: # WHEN we _delete_from_cache @@ -692,7 +692,7 @@ def test_is_missing_idempotency_key(): ) def test_default_no_raise_on_missing_idempotency_key(idempotency_config, persistence_store): # GIVEN a persistence_store with use_local_cache = False and event_key_jmespath = "body" - persistence_store.configure(idempotency_config) + persistence_store._configure(idempotency_config) assert persistence_store.use_local_cache is False assert "body" in persistence_store.event_key_jmespath @@ -708,7 +708,7 @@ def test_default_no_raise_on_missing_idempotency_key(idempotency_config, persist ) def test_raise_on_no_idempotency_key(idempotency_config, persistence_store): # GIVEN a persistence_store with raise_on_no_idempotency_key and no idempotency key in the request - persistence_store.configure(idempotency_config) + persistence_store._configure(idempotency_config) persistence_store.raise_on_no_idempotency_key = True assert persistence_store.use_local_cache is False assert "body" in persistence_store.event_key_jmespath @@ -724,7 +724,7 @@ def test_raise_on_no_idempotency_key(idempotency_config, persistence_store): def test_jmespath_with_powertools_json(persistence_store): # GIVEN an event_key_jmespath with powertools_json custom function config = IdempotencyConfig(event_key_jmespath="[requestContext.authorizer.claims.sub, powertools_json(body).id]") - persistence_store.configure(config) + persistence_store._configure(config) sub_attr_value = "cognito_user" key_attr_value = "some_key" expected_value = [sub_attr_value, key_attr_value] @@ -744,7 +744,7 @@ def test_jmespath_with_powertools_json(persistence_store): def test_custom_jmespath_function_overrides_builtin_functions(config_with_jmespath_options, persistence_store): # GIVEN an persistence store with a custom jmespath_options # AND use a builtin powertools custom function - persistence_store.configure(config_with_jmespath_options) + persistence_store._configure(config_with_jmespath_options) with pytest.raises(jmespath.exceptions.UnknownFunctionError, match="Unknown function: powertools_json()"): # WHEN calling _get_hashed_idempotency_key # THEN raise unknown function From 131bde483a2eac32c0f7c63d26c8f3d0e7ddbdb9 Mon Sep 17 00:00:00 2001 From: Michael Brewer Date: Thu, 4 Mar 2021 08:36:44 -0800 Subject: [PATCH 04/11] docs: Update docs --- docs/utilities/idempotency.md | 73 ++++++++++++++++++----------------- 1 file changed, 38 insertions(+), 35 deletions(-) diff --git a/docs/utilities/idempotency.md b/docs/utilities/idempotency.md index 8ea23bde5ae..a7e200b9128 100644 --- a/docs/utilities/idempotency.md +++ b/docs/utilities/idempotency.md @@ -83,18 +83,18 @@ this parameter, the entire event will be used as the key. === "app.py" - ```python hl_lines="2 6-9 11" + ```python hl_lines="2-4 8-9 11" import json - from aws_lambda_powertools.utilities.idempotency import DynamoDBPersistenceLayer, idempotent + from aws_lambda_powertools.utilities.idempotency import ( + IdempotencyConfig, DynamoDBPersistenceLayer, idempotent + ) # Treat everything under the "body" key in # the event json object as our payload - persistence_layer = DynamoDBPersistenceLayer( - table_name="IdempotencyTable", - event_key_jmespath="body", - ) + config = IdempotencyConfig(event_key_jmespath="body") + persistence_layer = DynamoDBPersistenceLayer(table_name="IdempotencyTable") - @idempotent(persistence_store=persistence_layer) + @idempotent(config=config, persistence_store=persistence_layer) def handler(event, context): body = json.loads(event['body']) payment = create_subscription_payment( @@ -174,9 +174,8 @@ change this window with the `expires_after_seconds` parameter: === "app.py" - ```python hl_lines="4" - DynamoDBPersistenceLayer( - table_name="IdempotencyTable", + ```python hl_lines="3" + IdempotencyConfig( event_key_jmespath="body", expires_after_seconds=5*60, # 5 minutes ) @@ -203,9 +202,8 @@ execution environment. You can change this with the `local_cache_max_items` para === "app.py" - ```python hl_lines="4 5" - DynamoDBPersistenceLayer( - table_name="IdempotencyTable", + ```python hl_lines="3 4" + IdempotencyConfig( event_key_jmespath="body", use_local_cache=True, local_cache_max_items=1000 @@ -224,16 +222,18 @@ idempotent invocations. === "app.py" - ```python hl_lines="6" - from aws_lambda_powertools.utilities.idempotency import DynamoDBPersistenceLayer, idempotent + ```python hl_lines="7" + from aws_lambda_powertools.utilities.idempotency import ( + IdempotencyConfig, DynamoDBPersistenceLayer, idempotent + ) - persistence_layer = DynamoDBPersistenceLayer( - table_name="IdempotencyTable", + config = IdempotencyConfig( event_key_jmespath="[userDetail, productId]", payload_validation_jmespath="amount" ) + persistence_layer = DynamoDBPersistenceLayer(table_name="IdempotencyTable") - @idempotent(persistence_store=persistence_layer) + @idempotent(config=config, persistence_store=persistence_layer) def handler(event, context): # Creating a subscription payment is a side # effect of calling this function! @@ -274,7 +274,7 @@ and we will raise `IdempotencyKeyError` if none was found. === "app.py" ```python hl_lines="4" - DynamoDBPersistenceLayer( + IdempotencyConfig( table_name="IdempotencyTable", event_key_jmespath="body", raise_on_no_idempotency_key=True, @@ -298,10 +298,9 @@ This example demonstrates changing the attribute names to custom values: === "app.py" - ```python hl_lines="4-8" + ```python hl_lines="3-7" persistence_layer = DynamoDBPersistenceLayer( table_name="IdempotencyTable", - event_key_jmespath="[userDetail, productId]", key_attr="idempotency_key", expiry_attr="expires_at", status_attr="current_status", @@ -316,35 +315,39 @@ or `boto3_session` parameters when constructing the persistence store. === "Custom session" - ```python hl_lines="1 4 8" + ```python hl_lines="1 7 10" import boto3 - from aws_lambda_powertools.utilities.idempotency import DynamoDBPersistenceLayer, idempotent + from aws_lambda_powertools.utilities.idempotency import ( + IdempotencyConfig, DynamoDBPersistenceLayer, idempotent + ) + config = IdempotencyConfig(event_key_jmespath="body") boto3_session = boto3.session.Session() persistence_layer = DynamoDBPersistenceLayer( table_name="IdempotencyTable", - event_key_jmespath="body", boto3_session=boto3_session ) - @idempotent(persistence_store=persistence_layer) + @idempotent(config=config, persistence_store=persistence_layer) def handler(event, context): ... ``` === "Custom config" - ```python hl_lines="1 4 8" + ```python hl_lines="1 7 10" from botocore.config import Config - from aws_lambda_powertools.utilities.idempotency import DynamoDBPersistenceLayer, idempotent + from aws_lambda_powertools.utilities.idempotency import ( + IdempotencyConfig, DynamoDBPersistenceLayer, idempotent + ) + config = IdempotencyConfig(event_key_jmespath="body") boto_config = Config() persistence_layer = DynamoDBPersistenceLayer( table_name="IdempotencyTable", - event_key_jmespath="body", boto_config=boto_config ) - @idempotent(persistence_store=persistence_layer) + @idempotent(config=config, persistence_store=persistence_layer) def handler(event, context): ... ``` @@ -372,15 +375,15 @@ The idempotency utility can be used with the `validator` decorator. Ensure that ```python hl_lines="9 10" from aws_lambda_powertools.utilities.validation import validator, envelopes - from aws_lambda_powertools.utilities.idempotency.idempotency import idempotent - - persistence_layer = DynamoDBPersistenceLayer( - table_name="IdempotencyTable", - event_key_jmespath="[message, username]", + from aws_lambda_powertools.utilities.idempotency import ( + IdempotencyConfig, DynamoDBPersistenceLayer, idempotent ) + config = IdempotencyConfig(event_key_jmespath="[message, username]") + persistence_layer = DynamoDBPersistenceLayer(table_name="IdempotencyTable") + @validator(envelope=envelopes.API_GATEWAY_HTTP) - @idempotent(persistence_store=persistence_layer) + @idempotent(config=config, persistence_store=persistence_layer) def lambda_handler(event, context): cause_some_side_effects(event['username') return {"message": event['message'], "statusCode": 200} From 07e8a063556d72527193b9527d8f64cc601be630 Mon Sep 17 00:00:00 2001 From: Heitor Lessa Date: Thu, 4 Mar 2021 19:23:43 +0100 Subject: [PATCH 05/11] docs(batch): add example on how to integrate with sentry.io (#308) --- docs/utilities/batch.md | 156 ++++++++++++++++++++++++++-------------- 1 file changed, 104 insertions(+), 52 deletions(-) diff --git a/docs/utilities/batch.md b/docs/utilities/batch.md index aa284e7f38b..ca4606e0f40 100644 --- a/docs/utilities/batch.md +++ b/docs/utilities/batch.md @@ -5,13 +5,13 @@ description: Utility The SQS batch processing utility provides a way to handle partial failures when processing batches of messages from SQS. -**Key Features** +## Key Features * Prevent successfully processed messages being returned to SQS * Simple interface for individually processing messages from a batch * Build your own batch processor using the base classes -**Background** +## Background When using SQS as a Lambda event source mapping, Lambda functions are triggered with a batch of messages from SQS. @@ -25,35 +25,76 @@ are returned to the queue. More details on how Lambda works with SQS can be found in the [AWS documentation](https://docs.aws.amazon.com/lambda/latest/dg/with-sqs.html) +## Getting started -**IAM Permissions** +### IAM Permissions -This utility requires additional permissions to work as expected. Lambda functions using this utility require the `sqs:DeleteMessageBatch` permission. +Before your use this utility, your AWS Lambda function must have `sqs:DeleteMessageBatch` permission to delete successful messages directly from the queue. -## Processing messages from SQS +> Example using AWS Serverless Application Model (SAM) -You can use either **[sqs_batch_processor](#sqs_batch_processor-decorator)** decorator, or **[PartialSQSProcessor](#partialsqsprocessor-context-manager)** as a context manager. +=== "template.yml" + ```yaml hl_lines="2-3 12-15" + Resources: + MyQueue: + Type: AWS::SQS::Queue -They have nearly the same behaviour when it comes to processing messages from the batch: + HelloWorldFunction: + Type: AWS::Serverless::Function + Properties: + Runtime: python3.8 + Environment: + Variables: + POWERTOOLS_SERVICE_NAME: example + Policies: + - SQSPollerPolicy: + QueueName: + !GetAtt MyQueue.QueueName + ``` -* **Entire batch has been successfully processed**, where your Lambda handler returned successfully, we will let SQS delete the batch to optimize your cost -* **Entire Batch has been partially processed successfully**, where exceptions were raised within your `record handler`, we will: - - **1)** Delete successfully processed messages from the queue by directly calling `sqs:DeleteMessageBatch` - - **2)** Raise `SQSBatchProcessingError` to ensure failed messages return to your SQS queue +### Processing messages from SQS -The only difference is that **PartialSQSProcessor** will give you access to processed messages if you need. +You can use either **[sqs_batch_processor](#sqs_batch_processor-decorator)** decorator, or **[PartialSQSProcessor](#partialsqsprocessor-context-manager)** as a context manager if you'd like access to the processed results. -## Record Handler +You need to create a function to handle each record from the batch - We call it `record_handler` from here on. -Both decorator and context managers require an explicit function to process the batch of messages - namely `record_handler` parameter. +=== "Decorator" -This function is responsible for processing each individual message from the batch, and to raise an exception if unable to process any of the messages sent. + ```python hl_lines="3 6" + from aws_lambda_powertools.utilities.batch import sqs_batch_processor -**Any non-exception/successful return from your record handler function** will instruct both decorator and context manager to queue up each individual message for deletion. + def record_handler(record): + return do_something_with(record["body"]) -### sqs_batch_processor decorator + @sqs_batch_processor(record_handler=record_handler) + def lambda_handler(event, context): + return {"statusCode": 200} + ``` +=== "Context manager" -When using this decorator, you need provide a function via `record_handler` param that will process individual messages from the batch - It should raise an exception if it is unable to process the record. + ```python hl_lines="3 9 11-12" + from aws_lambda_powertools.utilities.batch import PartialSQSProcessor + + def record_handler(record): + return_value = do_something_with(record["body"]) + return return_value + + def lambda_handler(event, context): + records = event["Records"] + processor = PartialSQSProcessor() + + with processor(records, record_handler) as proc: + result = proc.process() # Returns a list of all results from record_handler + + return result + ``` + +!!! tip + **Any non-exception/successful return from your record handler function** will instruct both decorator and context manager to queue up each individual message for deletion. + + If the entire batch succeeds, we let Lambda to proceed in deleting the records from the queue for cost reasons. + +### Partial failure mechanics All records in the batch will be passed to this handler for processing, even if exceptions are thrown - Here's the behaviour after completing the batch: @@ -61,29 +102,26 @@ All records in the batch will be passed to this handler for processing, even if * **Any unprocessed messages detected**, we will raise `SQSBatchProcessingError` to ensure failed messages return to your SQS queue !!! warning - You will not have accessed to the processed messages within the Lambda Handler - all processing logic will and should be performed by the record_handler function. + You will not have accessed to the **processed messages** within the Lambda Handler. -=== "app.py" + All processing logic will and should be performed by the `record_handler` function. - ```python - from aws_lambda_powertools.utilities.batch import sqs_batch_processor +## Advanced - def record_handler(record): - # This will be called for each individual message from a batch - # It should raise an exception if the message was not processed successfully - return_value = do_something_with(record["body"]) - return return_value +### Choosing between decorator and context manager - @sqs_batch_processor(record_handler=record_handler) - def lambda_handler(event, context): - return {"statusCode": 200} - ``` +They have nearly the same behaviour when it comes to processing messages from the batch: -### PartialSQSProcessor context manager +* **Entire batch has been successfully processed**, where your Lambda handler returned successfully, we will let SQS delete the batch to optimize your cost +* **Entire Batch has been partially processed successfully**, where exceptions were raised within your `record handler`, we will: + - **1)** Delete successfully processed messages from the queue by directly calling `sqs:DeleteMessageBatch` + - **2)** Raise `SQSBatchProcessingError` to ensure failed messages return to your SQS queue + +The only difference is that **PartialSQSProcessor** will give you access to processed messages if you need. -If you require access to the result of processed messages, you can use this context manager. +### Accessing processed messages -The result from calling `process()` on the context manager will be a list of all the return values from your `record_handler` function. +Use `PartialSQSProcessor` context manager to access a list of all return values from your `record_handler` function. === "app.py" @@ -91,11 +129,7 @@ The result from calling `process()` on the context manager will be a list of all from aws_lambda_powertools.utilities.batch import PartialSQSProcessor def record_handler(record): - # This will be called for each individual message from a batch - # It should raise an exception if the message was not processed successfully - return_value = do_something_with(record["body"]) - return return_value - + return do_something_with(record["body"]) def lambda_handler(event, context): records = event["Records"] @@ -108,7 +142,7 @@ The result from calling `process()` on the context manager will be a list of all return result ``` -## Passing custom boto3 config +### Passing custom boto3 config If you need to pass custom configuration such as region to the SDK, you can pass your own [botocore config object](https://botocore.amazonaws.com/v1/documentation/api/latest/reference/config.html) to the `sqs_batch_processor` decorator: @@ -159,14 +193,15 @@ the `sqs_batch_processor` decorator: ``` -## Suppressing exceptions +### Suppressing exceptions If you want to disable the default behavior where `SQSBatchProcessingError` is raised if there are any errors, you can pass the `suppress_exception` boolean argument. === "Decorator" - ```python hl_lines="2" - ... + ```python hl_lines="3" + from aws_lambda_powertools.utilities.batch import sqs_batch_processor + @sqs_batch_processor(record_handler=record_handler, config=config, suppress_exception=True) def lambda_handler(event, context): return {"statusCode": 200} @@ -174,15 +209,16 @@ If you want to disable the default behavior where `SQSBatchProcessingError` is r === "Context manager" - ```python hl_lines="2" - ... + ```python hl_lines="3" + from aws_lambda_powertools.utilities.batch import PartialSQSProcessor + processor = PartialSQSProcessor(config=config, suppress_exception=True) with processor(records, record_handler): result = processor.process() ``` -## Create your own partial processor +### Create your own partial processor You can create your own partial batch processor by inheriting the `BasePartialProcessor` class, and implementing `_prepare()`, `_clean()` and `_process_record()`. @@ -192,11 +228,9 @@ You can create your own partial batch processor by inheriting the `BasePartialPr You can then use this class as a context manager, or pass it to `batch_processor` to use as a decorator on your Lambda handler function. -**Example:** - === "custom_processor.py" - ```python + ```python hl_lines="3 9 24 30 37 57" from random import randint from aws_lambda_powertools.utilities.batch import BasePartialProcessor, batch_processor @@ -223,14 +257,12 @@ You can then use this class as a context manager, or pass it to `batch_processor def _prepare(self): # It's called once, *before* processing # Creates table resource and clean previous results - # E.g.: self.ddb_table = boto3.resource("dynamodb").Table(self.table_name) self.success_messages.clear() def _clean(self): # It's called once, *after* closing processing all records (closing the context manager) # Here we're sending, at once, all successful messages to a ddb table - # E.g.: with ddb_table.batch_writer() as batch: for result in self.success_messages: batch.put_item(Item=result) @@ -239,7 +271,6 @@ You can then use this class as a context manager, or pass it to `batch_processor # It handles how your record is processed # Here we're keeping the status of each run # where self.handler is the record_handler function passed as an argument - # E.g.: try: result = self.handler(record) # record_handler passed to decorator/context manager return self.success_handler(record, result) @@ -260,3 +291,24 @@ You can then use this class as a context manager, or pass it to `batch_processor def lambda_handler(event, context): return {"statusCode": 200} ``` + +### Integrating exception handling with Sentry.io + +When using Sentry.io for error monitoring, you can override `failure_handler` to include to capture each processing exception: + +> Credits to [Charles-Axel Dein](https://github.com/awslabs/aws-lambda-powertools-python/issues/293#issuecomment-781961732) + +=== "sentry_integration.py" + + ```python hl_lines="4 7-8" + from typing import Tuple + + from aws_lambda_powertools.utilities.batch import PartialSQSProcessor + from sentry_sdk import capture_exception + + class SQSProcessor(PartialSQSProcessor): + def failure_handler(self, record: Event, exception: Tuple) -> Tuple: # type: ignore + capture_exception() # send exception to Sentry + logger.exception("got exception while processing SQS message") + return super().failure_handler(record, exception) # type: ignore + ``` From 7d4304bab5d8858a99063dcb51618065d20266de Mon Sep 17 00:00:00 2001 From: Michael Brewer Date: Thu, 4 Mar 2021 15:06:19 -0800 Subject: [PATCH 06/11] chore(docs): Update the docs --- docs/utilities/idempotency.md | 42 ++++++++++++++++++++++++++++++----- 1 file changed, 36 insertions(+), 6 deletions(-) diff --git a/docs/utilities/idempotency.md b/docs/utilities/idempotency.md index a7e200b9128..78880d02f7d 100644 --- a/docs/utilities/idempotency.md +++ b/docs/utilities/idempotency.md @@ -75,7 +75,7 @@ storage layer, so you'll need to create a table first. You can quickly start by initializing the `DynamoDBPersistenceLayer` class outside the Lambda handler, and using it with the `idempotent` decorator on your lambda handler. The only required parameter is `table_name`, but you likely -want to specify `event_key_jmespath` as well. +want to specify `event_key_jmespath` via `IdempotencyConfig`. `event_key_jmespath`: A JMESpath expression which will be used to extract the payload from the event your Lambda handler is called with. This payload will be used as the key to decide if future invocations are duplicates. If you don't pass @@ -100,7 +100,7 @@ this parameter, the entire event will be used as the key. payment = create_subscription_payment( user=body['user'], product=body['product_id'] - ) + ) ... return {"message": "success", "statusCode": 200, "payment_id": payment.id} ``` @@ -273,12 +273,42 @@ and we will raise `IdempotencyKeyError` if none was found. === "app.py" - ```python hl_lines="4" - IdempotencyConfig( - table_name="IdempotencyTable", - event_key_jmespath="body", + ```python hl_lines="8" + from aws_lambda_powertools.utilities.idempotency import ( + IdempotencyConfig, DynamoDBPersistenceLayer, idempotent + ) + + # Requires "user"."uid" and from the "body" json parsed "order_id" to be present + config = IdempotencyConfig( + event_key_jmespath="[user.uid, powertools_json(body).order_id]", raise_on_no_idempotency_key=True, ) + persistence_layer = DynamoDBPersistenceLayer(table_name="IdempotencyTable") + + @idempotent(config=config, persistence_store=persistence_layer) + def handler(event, context): + pass + ``` +=== "Success Event" + + ```json + { + "user": { + "uid": "BB0D045C-8878-40C8-889E-38B3CB0A61B1", + "name": "Foo" + }, + "body": "{\"order_id\": 10000}" + } + ``` +=== "Failure Event" + + ```json + { + "user": { + "name": "Foo" + }, + "body": "{\"total_amount\": 10000}" + } ``` ### Changing dynamoDB attribute names From 0fc4ee2266c00f4f4ad4d5f75ca7a92f359da237 Mon Sep 17 00:00:00 2001 From: Michael Brewer Date: Thu, 4 Mar 2021 21:44:48 -0800 Subject: [PATCH 07/11] fix(tests): Fix coverage-html and various update --- Makefile | 2 +- .../utilities/idempotency/idempotency.py | 3 +- .../utilities/idempotency/persistence/base.py | 2 +- docs/utilities/idempotency.md | 2 +- tests/functional/idempotency/conftest.py | 2 +- .../idempotency/test_idempotency.py | 134 +++++++++++------- 6 files changed, 86 insertions(+), 59 deletions(-) diff --git a/Makefile b/Makefile index e56eb4bb266..8532d9fa6a4 100644 --- a/Makefile +++ b/Makefile @@ -21,7 +21,7 @@ test: poetry run pytest --cache-clear tests/performance coverage-html: - poetry run pytest --cov-report html + poetry run pytest -m "not perf" --cov-report html pr: lint test security-baseline complexity-baseline diff --git a/aws_lambda_powertools/utilities/idempotency/idempotency.py b/aws_lambda_powertools/utilities/idempotency/idempotency.py index 9ad0bdfdff3..30d449af962 100644 --- a/aws_lambda_powertools/utilities/idempotency/idempotency.py +++ b/aws_lambda_powertools/utilities/idempotency/idempotency.py @@ -55,9 +55,10 @@ def idempotent( >>> idempotent, DynamoDBPersistenceLayer, IdempotencyConfig >>> ) >>> + >>> idem_config=IdempotencyConfig(event_key_jmespath="body") >>> persistence_layer = DynamoDBPersistenceLayer(table_name="idempotency_store") >>> - >>> @idempotent(persistence_store=persistence_layer, config=IdempotencyConfig(event_key_jmespath="body")) + >>> @idempotent(config=idem_config, persistence_store=persistence_layer) >>> def handler(event, context): >>> return {"StatusCode": 200} """ diff --git a/aws_lambda_powertools/utilities/idempotency/persistence/base.py b/aws_lambda_powertools/utilities/idempotency/persistence/base.py index 444b6bcc070..ee6fca1e77b 100644 --- a/aws_lambda_powertools/utilities/idempotency/persistence/base.py +++ b/aws_lambda_powertools/utilities/idempotency/persistence/base.py @@ -132,7 +132,7 @@ def _configure(self, config: IdempotencyConfig) -> None: Idempotency configuration settings """ if self.configured: - # Prevent being reconfigured. + # Prevent being reconfigured multiple times return self.configured = True diff --git a/docs/utilities/idempotency.md b/docs/utilities/idempotency.md index 78880d02f7d..f231580b606 100644 --- a/docs/utilities/idempotency.md +++ b/docs/utilities/idempotency.md @@ -75,7 +75,7 @@ storage layer, so you'll need to create a table first. You can quickly start by initializing the `DynamoDBPersistenceLayer` class outside the Lambda handler, and using it with the `idempotent` decorator on your lambda handler. The only required parameter is `table_name`, but you likely -want to specify `event_key_jmespath` via `IdempotencyConfig`. +want to specify `event_key_jmespath` via `IdempotencyConfig` class. `event_key_jmespath`: A JMESpath expression which will be used to extract the payload from the event your Lambda handler is called with. This payload will be used as the key to decide if future invocations are duplicates. If you don't pass diff --git a/tests/functional/idempotency/conftest.py b/tests/functional/idempotency/conftest.py index 004de5e52bf..532d551ef40 100644 --- a/tests/functional/idempotency/conftest.py +++ b/tests/functional/idempotency/conftest.py @@ -166,7 +166,7 @@ def idempotency_config(config, request, default_jmespath): @pytest.fixture def config_without_jmespath(config, request): - return IdempotencyConfig(use_local_cache=request.param["use_local_cache"],) + return IdempotencyConfig(use_local_cache=request.param["use_local_cache"]) @pytest.fixture diff --git a/tests/functional/idempotency/test_idempotency.py b/tests/functional/idempotency/test_idempotency.py index ccebf7c1ba1..3031a522b87 100644 --- a/tests/functional/idempotency/test_idempotency.py +++ b/tests/functional/idempotency/test_idempotency.py @@ -7,7 +7,7 @@ import pytest from botocore import stub -from aws_lambda_powertools.utilities.idempotency import IdempotencyConfig +from aws_lambda_powertools.utilities.idempotency import DynamoDBPersistenceLayer, IdempotencyConfig from aws_lambda_powertools.utilities.idempotency.exceptions import ( IdempotencyAlreadyInProgressError, IdempotencyInconsistentStateError, @@ -27,8 +27,8 @@ # enabled, and one without. @pytest.mark.parametrize("idempotency_config", [{"use_local_cache": False}, {"use_local_cache": True}], indirect=True) def test_idempotent_lambda_already_completed( - idempotency_config, - persistence_store, + idempotency_config: IdempotencyConfig, + persistence_store: DynamoDBPersistenceLayer, lambda_apigw_event, timestamp_future, hashed_idempotency_key, @@ -58,7 +58,7 @@ def test_idempotent_lambda_already_completed( stubber.add_response("get_item", ddb_response, expected_params) stubber.activate() - @idempotent(persistence_store=persistence_store, config=idempotency_config) + @idempotent(config=idempotency_config, persistence_store=persistence_store) def lambda_handler(event, context): raise Exception @@ -71,7 +71,12 @@ def lambda_handler(event, context): @pytest.mark.parametrize("idempotency_config", [{"use_local_cache": False}, {"use_local_cache": True}], indirect=True) def test_idempotent_lambda_in_progress( - idempotency_config, persistence_store, lambda_apigw_event, lambda_response, timestamp_future, hashed_idempotency_key + idempotency_config: IdempotencyConfig, + persistence_store: DynamoDBPersistenceLayer, + lambda_apigw_event, + lambda_response, + timestamp_future, + hashed_idempotency_key, ): """ Test idempotent decorator where lambda_handler is already processing an event with matching event key @@ -96,7 +101,7 @@ def test_idempotent_lambda_in_progress( stubber.add_response("get_item", ddb_response, expected_params) stubber.activate() - @idempotent(persistence_store=persistence_store, config=idempotency_config) + @idempotent(config=idempotency_config, persistence_store=persistence_store) def lambda_handler(event, context): return lambda_response @@ -114,8 +119,8 @@ def lambda_handler(event, context): @pytest.mark.skipif(sys.version_info < (3, 8), reason="issue with pytest mock lib for < 3.8") @pytest.mark.parametrize("idempotency_config", [{"use_local_cache": True}], indirect=True) def test_idempotent_lambda_in_progress_with_cache( - idempotency_config, - persistence_store, + idempotency_config: IdempotencyConfig, + persistence_store: DynamoDBPersistenceLayer, lambda_apigw_event, lambda_response, timestamp_future, @@ -153,7 +158,7 @@ def test_idempotent_lambda_in_progress_with_cache( stubber.add_response("get_item", copy.deepcopy(ddb_response), copy.deepcopy(expected_params)) stubber.activate() - @idempotent(persistence_store=persistence_store, config=idempotency_config) + @idempotent(config=idempotency_config, persistence_store=persistence_store) def lambda_handler(event, context): return lambda_response @@ -178,8 +183,8 @@ def lambda_handler(event, context): @pytest.mark.parametrize("idempotency_config", [{"use_local_cache": False}, {"use_local_cache": True}], indirect=True) def test_idempotent_lambda_first_execution( - idempotency_config, - persistence_store, + idempotency_config: IdempotencyConfig, + persistence_store: DynamoDBPersistenceLayer, lambda_apigw_event, expected_params_update_item, expected_params_put_item, @@ -199,7 +204,7 @@ def test_idempotent_lambda_first_execution( stubber.add_response("update_item", ddb_response, expected_params_update_item) stubber.activate() - @idempotent(persistence_store=persistence_store, config=idempotency_config) + @idempotent(config=idempotency_config, persistence_store=persistence_store) def lambda_handler(event, context): return lambda_response @@ -212,9 +217,9 @@ def lambda_handler(event, context): @pytest.mark.skipif(sys.version_info < (3, 8), reason="issue with pytest mock lib for < 3.8") @pytest.mark.parametrize("idempotency_config", [{"use_local_cache": True}], indirect=True) def test_idempotent_lambda_first_execution_cached( - idempotency_config, - persistence_store, - lambda_apigw_event, + idempotency_config: IdempotencyConfig, + persistence_store: DynamoDBPersistenceLayer, + lambda_apigw_event: DynamoDBPersistenceLayer, expected_params_update_item, expected_params_put_item, lambda_response, @@ -225,7 +230,6 @@ def test_idempotent_lambda_first_execution_cached( Test idempotent decorator when lambda is executed with an event with a previously unknown event key. Ensure result is cached locally on the persistence store instance. """ - persistence_store._configure(idempotency_config) save_to_cache_spy = mocker.spy(persistence_store, "_save_to_cache") retrieve_from_cache_spy = mocker.spy(persistence_store, "_retrieve_from_cache") stubber = stub.Stubber(persistence_store.table.meta.client) @@ -235,7 +239,7 @@ def test_idempotent_lambda_first_execution_cached( stubber.add_response("update_item", ddb_response, expected_params_update_item) stubber.activate() - @idempotent(persistence_store=persistence_store, config=idempotency_config) + @idempotent(config=idempotency_config, persistence_store=persistence_store) def lambda_handler(event, context): return lambda_response @@ -258,8 +262,8 @@ def lambda_handler(event, context): @pytest.mark.parametrize("idempotency_config", [{"use_local_cache": False}, {"use_local_cache": True}], indirect=True) def test_idempotent_lambda_expired( - idempotency_config, - persistence_store, + idempotency_config: IdempotencyConfig, + persistence_store: DynamoDBPersistenceLayer, lambda_apigw_event, timestamp_expired, lambda_response, @@ -280,7 +284,7 @@ def test_idempotent_lambda_expired( stubber.add_response("update_item", ddb_response, expected_params_update_item) stubber.activate() - @idempotent(persistence_store=persistence_store, config=idempotency_config) + @idempotent(config=idempotency_config, persistence_store=persistence_store) def lambda_handler(event, context): return lambda_response @@ -292,8 +296,8 @@ def lambda_handler(event, context): @pytest.mark.parametrize("idempotency_config", [{"use_local_cache": False}, {"use_local_cache": True}], indirect=True) def test_idempotent_lambda_exception( - idempotency_config, - persistence_store, + idempotency_config: IdempotencyConfig, + persistence_store: DynamoDBPersistenceLayer, lambda_apigw_event, timestamp_future, lambda_response, @@ -317,7 +321,7 @@ def test_idempotent_lambda_exception( stubber.add_response("delete_item", ddb_response, expected_params_delete_item) stubber.activate() - @idempotent(persistence_store=persistence_store, config=idempotency_config) + @idempotent(config=idempotency_config, persistence_store=persistence_store) def lambda_handler(event, context): raise Exception("Something went wrong!") @@ -332,8 +336,8 @@ def lambda_handler(event, context): "config_with_validation", [{"use_local_cache": False}, {"use_local_cache": True}], indirect=True ) def test_idempotent_lambda_already_completed_with_validation_bad_payload( - config_with_validation, - persistence_store, + config_with_validation: IdempotencyConfig, + persistence_store: DynamoDBPersistenceLayer, lambda_apigw_event, timestamp_future, lambda_response, @@ -361,7 +365,7 @@ def test_idempotent_lambda_already_completed_with_validation_bad_payload( stubber.add_response("get_item", ddb_response, expected_params) stubber.activate() - @idempotent(persistence_store=persistence_store, config=config_with_validation) + @idempotent(config=config_with_validation, persistence_store=persistence_store) def lambda_handler(event, context): return lambda_response @@ -375,8 +379,8 @@ def lambda_handler(event, context): @pytest.mark.parametrize("idempotency_config", [{"use_local_cache": False}, {"use_local_cache": True}], indirect=True) def test_idempotent_lambda_expired_during_request( - idempotency_config, - persistence_store, + idempotency_config: IdempotencyConfig, + persistence_store: DynamoDBPersistenceLayer, lambda_apigw_event, timestamp_expired, lambda_response, @@ -417,7 +421,7 @@ def test_idempotent_lambda_expired_during_request( stubber.activate() - @idempotent(persistence_store=persistence_store, config=idempotency_config) + @idempotent(config=idempotency_config, persistence_store=persistence_store) def lambda_handler(event, context): return lambda_response @@ -431,8 +435,8 @@ def lambda_handler(event, context): @pytest.mark.parametrize("idempotency_config", [{"use_local_cache": False}, {"use_local_cache": True}], indirect=True) def test_idempotent_persistence_exception_deleting( - idempotency_config, - persistence_store, + idempotency_config: IdempotencyConfig, + persistence_store: DynamoDBPersistenceLayer, lambda_apigw_event, timestamp_future, lambda_response, @@ -451,7 +455,7 @@ def test_idempotent_persistence_exception_deleting( stubber.add_client_error("delete_item", "UnrecoverableError") stubber.activate() - @idempotent(persistence_store=persistence_store, config=idempotency_config) + @idempotent(config=idempotency_config, persistence_store=persistence_store) def lambda_handler(event, context): raise Exception("Something went wrong!") @@ -465,8 +469,8 @@ def lambda_handler(event, context): @pytest.mark.parametrize("idempotency_config", [{"use_local_cache": False}, {"use_local_cache": True}], indirect=True) def test_idempotent_persistence_exception_updating( - idempotency_config, - persistence_store, + idempotency_config: IdempotencyConfig, + persistence_store: DynamoDBPersistenceLayer, lambda_apigw_event, timestamp_future, lambda_response, @@ -485,7 +489,7 @@ def test_idempotent_persistence_exception_updating( stubber.add_client_error("update_item", "UnrecoverableError") stubber.activate() - @idempotent(persistence_store=persistence_store, config=idempotency_config) + @idempotent(config=idempotency_config, persistence_store=persistence_store) def lambda_handler(event, context): return {"message": "success!"} @@ -499,8 +503,8 @@ def lambda_handler(event, context): @pytest.mark.parametrize("idempotency_config", [{"use_local_cache": False}, {"use_local_cache": True}], indirect=True) def test_idempotent_persistence_exception_getting( - idempotency_config, - persistence_store, + idempotency_config: IdempotencyConfig, + persistence_store: DynamoDBPersistenceLayer, lambda_apigw_event, timestamp_future, lambda_response, @@ -517,7 +521,7 @@ def test_idempotent_persistence_exception_getting( stubber.add_client_error("get_item", "UnexpectedException") stubber.activate() - @idempotent(persistence_store=persistence_store, config=idempotency_config) + @idempotent(config=idempotency_config, persistence_store=persistence_store) def lambda_handler(event, context): return {"message": "success!"} @@ -533,8 +537,8 @@ def lambda_handler(event, context): "config_with_validation", [{"use_local_cache": False}, {"use_local_cache": True}], indirect=True ) def test_idempotent_lambda_first_execution_with_validation( - config_with_validation, - persistence_store, + config_with_validation: IdempotencyConfig, + persistence_store: DynamoDBPersistenceLayer, lambda_apigw_event, expected_params_update_item_with_validation, expected_params_put_item_with_validation, @@ -552,8 +556,8 @@ def test_idempotent_lambda_first_execution_with_validation( stubber.add_response("update_item", ddb_response, expected_params_update_item_with_validation) stubber.activate() - @idempotent(persistence_store=persistence_store, config=config_with_validation) - def lambda_handler(lambda_apigw_event, context): + @idempotent(config=config_with_validation, persistence_store=persistence_store) + def lambda_handler(event, context): return lambda_response lambda_handler(lambda_apigw_event, {}) @@ -566,8 +570,8 @@ def lambda_handler(lambda_apigw_event, context): "config_without_jmespath", [{"use_local_cache": False}, {"use_local_cache": True}], indirect=True ) def test_idempotent_lambda_with_validator_util( - config_without_jmespath, - persistence_store, + config_without_jmespath: IdempotencyConfig, + persistence_store: DynamoDBPersistenceLayer, lambda_apigw_event, timestamp_future, serialized_lambda_response, @@ -600,7 +604,7 @@ def test_idempotent_lambda_with_validator_util( stubber.activate() @validator(envelope=envelopes.API_GATEWAY_HTTP) - @idempotent(persistence_store=persistence_store, config=config_without_jmespath) + @idempotent(config=config_without_jmespath, persistence_store=persistence_store) def lambda_handler(event, context): mock_function() return "shouldn't get here!" @@ -622,7 +626,9 @@ def test_data_record_invalid_status_value(): @pytest.mark.parametrize("idempotency_config", [{"use_local_cache": True}], indirect=True) -def test_in_progress_never_saved_to_cache(idempotency_config, persistence_store): +def test_in_progress_never_saved_to_cache( + idempotency_config: IdempotencyConfig, persistence_store: DynamoDBPersistenceLayer +): # GIVEN a data record with status "INPROGRESS" # and persistence_store has use_local_cache = True persistence_store._configure(idempotency_config) @@ -636,7 +642,7 @@ def test_in_progress_never_saved_to_cache(idempotency_config, persistence_store) @pytest.mark.parametrize("idempotency_config", [{"use_local_cache": False}], indirect=True) -def test_user_local_disabled(idempotency_config, persistence_store): +def test_user_local_disabled(idempotency_config: IdempotencyConfig, persistence_store: DynamoDBPersistenceLayer): # GIVEN a persistence_store with use_local_cache = False persistence_store._configure(idempotency_config) @@ -656,7 +662,9 @@ def test_user_local_disabled(idempotency_config, persistence_store): @pytest.mark.parametrize("idempotency_config", [{"use_local_cache": True}], indirect=True) -def test_delete_from_cache_when_empty(idempotency_config, persistence_store): +def test_delete_from_cache_when_empty( + idempotency_config: IdempotencyConfig, persistence_store: DynamoDBPersistenceLayer +): # GIVEN use_local_cache is True AND the local cache is empty persistence_store._configure(idempotency_config) @@ -690,7 +698,9 @@ def test_is_missing_idempotency_key(): @pytest.mark.parametrize( "idempotency_config", [{"use_local_cache": False, "event_key_jmespath": "body"}], indirect=True ) -def test_default_no_raise_on_missing_idempotency_key(idempotency_config, persistence_store): +def test_default_no_raise_on_missing_idempotency_key( + idempotency_config: IdempotencyConfig, persistence_store: DynamoDBPersistenceLayer +): # GIVEN a persistence_store with use_local_cache = False and event_key_jmespath = "body" persistence_store._configure(idempotency_config) assert persistence_store.use_local_cache is False @@ -706,7 +716,9 @@ def test_default_no_raise_on_missing_idempotency_key(idempotency_config, persist @pytest.mark.parametrize( "idempotency_config", [{"use_local_cache": False, "event_key_jmespath": "[body, x]"}], indirect=True ) -def test_raise_on_no_idempotency_key(idempotency_config, persistence_store): +def test_raise_on_no_idempotency_key( + idempotency_config: IdempotencyConfig, persistence_store: DynamoDBPersistenceLayer +): # GIVEN a persistence_store with raise_on_no_idempotency_key and no idempotency key in the request persistence_store._configure(idempotency_config) persistence_store.raise_on_no_idempotency_key = True @@ -721,10 +733,21 @@ def test_raise_on_no_idempotency_key(idempotency_config, persistence_store): assert "No data found to create a hashed idempotency_key" in str(excinfo.value) -def test_jmespath_with_powertools_json(persistence_store): +@pytest.mark.parametrize( + "idempotency_config", + [ + { + "use_local_cache": False, + "event_key_jmespath": "[requestContext.authorizer.claims.sub, powertools_json(body).id]", + } + ], + indirect=True, +) +def test_jmespath_with_powertools_json( + idempotency_config: IdempotencyConfig, persistence_store: DynamoDBPersistenceLayer +): # GIVEN an event_key_jmespath with powertools_json custom function - config = IdempotencyConfig(event_key_jmespath="[requestContext.authorizer.claims.sub, powertools_json(body).id]") - persistence_store._configure(config) + persistence_store._configure(idempotency_config) sub_attr_value = "cognito_user" key_attr_value = "some_key" expected_value = [sub_attr_value, key_attr_value] @@ -741,10 +764,13 @@ def test_jmespath_with_powertools_json(persistence_store): @pytest.mark.parametrize("config_with_jmespath_options", ["powertools_json(data).payload"], indirect=True) -def test_custom_jmespath_function_overrides_builtin_functions(config_with_jmespath_options, persistence_store): +def test_custom_jmespath_function_overrides_builtin_functions( + config_with_jmespath_options: IdempotencyConfig, persistence_store: DynamoDBPersistenceLayer +): # GIVEN an persistence store with a custom jmespath_options # AND use a builtin powertools custom function persistence_store._configure(config_with_jmespath_options) + with pytest.raises(jmespath.exceptions.UnknownFunctionError, match="Unknown function: powertools_json()"): # WHEN calling _get_hashed_idempotency_key # THEN raise unknown function From eb4787cf04edea0d3f9bdbca9583955eac92a377 Mon Sep 17 00:00:00 2001 From: Michael Brewer Date: Thu, 4 Mar 2021 21:51:56 -0800 Subject: [PATCH 08/11] refactor: Change back to configure --- .../utilities/idempotency/idempotency.py | 2 +- .../utilities/idempotency/persistence/base.py | 2 +- tests/functional/idempotency/test_idempotency.py | 14 +++++++------- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/aws_lambda_powertools/utilities/idempotency/idempotency.py b/aws_lambda_powertools/utilities/idempotency/idempotency.py index 30d449af962..235e5c884d6 100644 --- a/aws_lambda_powertools/utilities/idempotency/idempotency.py +++ b/aws_lambda_powertools/utilities/idempotency/idempotency.py @@ -106,7 +106,7 @@ def __init__( persistence_store : BasePersistenceLayer Instance of persistence layer to store idempotency records """ - persistence_store._configure(config) + persistence_store.configure(config) self.persistence_store = persistence_store self.context = context self.event = event diff --git a/aws_lambda_powertools/utilities/idempotency/persistence/base.py b/aws_lambda_powertools/utilities/idempotency/persistence/base.py index ee6fca1e77b..58f67a292e7 100644 --- a/aws_lambda_powertools/utilities/idempotency/persistence/base.py +++ b/aws_lambda_powertools/utilities/idempotency/persistence/base.py @@ -122,7 +122,7 @@ def __init__(self): self._cache: Optional[LRUDict] = None self.hash_function = None - def _configure(self, config: IdempotencyConfig) -> None: + def configure(self, config: IdempotencyConfig) -> None: """ Initialize the base persistence layer from the configuration settings diff --git a/tests/functional/idempotency/test_idempotency.py b/tests/functional/idempotency/test_idempotency.py index 3031a522b87..999b34fc8f6 100644 --- a/tests/functional/idempotency/test_idempotency.py +++ b/tests/functional/idempotency/test_idempotency.py @@ -631,7 +631,7 @@ def test_in_progress_never_saved_to_cache( ): # GIVEN a data record with status "INPROGRESS" # and persistence_store has use_local_cache = True - persistence_store._configure(idempotency_config) + persistence_store.configure(idempotency_config) data_record = DataRecord("key", status="INPROGRESS") # WHEN saving to local cache @@ -644,7 +644,7 @@ def test_in_progress_never_saved_to_cache( @pytest.mark.parametrize("idempotency_config", [{"use_local_cache": False}], indirect=True) def test_user_local_disabled(idempotency_config: IdempotencyConfig, persistence_store: DynamoDBPersistenceLayer): # GIVEN a persistence_store with use_local_cache = False - persistence_store._configure(idempotency_config) + persistence_store.configure(idempotency_config) # WHEN calling any local cache options data_record = DataRecord("key", status="COMPLETED") @@ -666,7 +666,7 @@ def test_delete_from_cache_when_empty( idempotency_config: IdempotencyConfig, persistence_store: DynamoDBPersistenceLayer ): # GIVEN use_local_cache is True AND the local cache is empty - persistence_store._configure(idempotency_config) + persistence_store.configure(idempotency_config) try: # WHEN we _delete_from_cache @@ -702,7 +702,7 @@ def test_default_no_raise_on_missing_idempotency_key( idempotency_config: IdempotencyConfig, persistence_store: DynamoDBPersistenceLayer ): # GIVEN a persistence_store with use_local_cache = False and event_key_jmespath = "body" - persistence_store._configure(idempotency_config) + persistence_store.configure(idempotency_config) assert persistence_store.use_local_cache is False assert "body" in persistence_store.event_key_jmespath @@ -720,7 +720,7 @@ def test_raise_on_no_idempotency_key( idempotency_config: IdempotencyConfig, persistence_store: DynamoDBPersistenceLayer ): # GIVEN a persistence_store with raise_on_no_idempotency_key and no idempotency key in the request - persistence_store._configure(idempotency_config) + persistence_store.configure(idempotency_config) persistence_store.raise_on_no_idempotency_key = True assert persistence_store.use_local_cache is False assert "body" in persistence_store.event_key_jmespath @@ -747,7 +747,7 @@ def test_jmespath_with_powertools_json( idempotency_config: IdempotencyConfig, persistence_store: DynamoDBPersistenceLayer ): # GIVEN an event_key_jmespath with powertools_json custom function - persistence_store._configure(idempotency_config) + persistence_store.configure(idempotency_config) sub_attr_value = "cognito_user" key_attr_value = "some_key" expected_value = [sub_attr_value, key_attr_value] @@ -769,7 +769,7 @@ def test_custom_jmespath_function_overrides_builtin_functions( ): # GIVEN an persistence store with a custom jmespath_options # AND use a builtin powertools custom function - persistence_store._configure(config_with_jmespath_options) + persistence_store.configure(config_with_jmespath_options) with pytest.raises(jmespath.exceptions.UnknownFunctionError, match="Unknown function: powertools_json()"): # WHEN calling _get_hashed_idempotency_key From b041f3fb904781da9816c1f412691a362ec78a70 Mon Sep 17 00:00:00 2001 From: Michael Brewer Date: Thu, 4 Mar 2021 22:18:39 -0800 Subject: [PATCH 09/11] chore: Hide missing code coverage --- aws_lambda_powertools/tracing/extensions.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/aws_lambda_powertools/tracing/extensions.py b/aws_lambda_powertools/tracing/extensions.py index 2bb0125e841..6c641238c98 100644 --- a/aws_lambda_powertools/tracing/extensions.py +++ b/aws_lambda_powertools/tracing/extensions.py @@ -8,8 +8,8 @@ def aiohttp_trace_config(): TraceConfig aiohttp trace config """ - from aws_xray_sdk.ext.aiohttp.client import aws_xray_trace_config + from aws_xray_sdk.ext.aiohttp.client import aws_xray_trace_config # pragma: no cover - aws_xray_trace_config.__doc__ = "aiohttp extension for X-Ray (aws_xray_trace_config)" + aws_xray_trace_config.__doc__ = "aiohttp extension for X-Ray (aws_xray_trace_config)" # pragma: no cover - return aws_xray_trace_config() + return aws_xray_trace_config() # pragma: no cover From 8c2350e04069a8a01df79e8f91c58eb0260cb92d Mon Sep 17 00:00:00 2001 From: Michael Brewer Date: Thu, 4 Mar 2021 22:31:47 -0800 Subject: [PATCH 10/11] chore: bump ci --- docs/utilities/idempotency.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/utilities/idempotency.md b/docs/utilities/idempotency.md index f231580b606..784f6597a23 100644 --- a/docs/utilities/idempotency.md +++ b/docs/utilities/idempotency.md @@ -305,7 +305,7 @@ and we will raise `IdempotencyKeyError` if none was found. ```json { "user": { - "name": "Foo" + "name": "Joe Bloggs" }, "body": "{\"total_amount\": 10000}" } From 1254b06b73f2f9a6a029f8b1944d8401341eff3e Mon Sep 17 00:00:00 2001 From: Michael Brewer Date: Thu, 4 Mar 2021 22:52:36 -0800 Subject: [PATCH 11/11] chore: bump ci --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 8532d9fa6a4..d11ea72779f 100644 --- a/Makefile +++ b/Makefile @@ -21,7 +21,7 @@ test: poetry run pytest --cache-clear tests/performance coverage-html: - poetry run pytest -m "not perf" --cov-report html + poetry run pytest -m "not perf" --cov-report=html pr: lint test security-baseline complexity-baseline