feat(idempotency): makes customers unit testing easier (#719)

Co-authored-by: Heitor Lessa <[email protected]>

Author: Tom McCarthy

aws_lambda_powertools/shared/constants.py

@@ -21,3 +21,5 @@
 
 XRAY_SDK_MODULE: str = "aws_xray_sdk"
 XRAY_SDK_CORE_MODULE: str = "aws_xray_sdk.core"
+
+IDEMPOTENCY_DISABLED_ENV: str = "POWERTOOLS_IDEMPOTENCY_DISABLED"

aws_lambda_powertools/utilities/idempotency/idempotency.py

@@ -3,9 +3,11 @@
 """
 import functools
 import logging
+import os
 from typing import Any, Callable, Dict, Optional, cast
 
 from aws_lambda_powertools.middleware_factory import lambda_handler_decorator
+from aws_lambda_powertools.shared.constants import IDEMPOTENCY_DISABLED_ENV
 from aws_lambda_powertools.shared.types import AnyCallableT
 from aws_lambda_powertools.utilities.idempotency.base import IdempotencyHandler
 from aws_lambda_powertools.utilities.idempotency.config import IdempotencyConfig
@@ -56,6 +58,9 @@ def idempotent(
         >>>     return {"StatusCode": 200}
     """
 
+    if os.getenv(IDEMPOTENCY_DISABLED_ENV):
+        return handler(event, context)
+
     config = config or IdempotencyConfig()
     args = event, context
     idempotency_handler = IdempotencyHandler(
@@ -122,6 +127,9 @@ def process_order(customer_id: str, order: dict, **kwargs):
 
     @functools.wraps(function)
     def decorate(*args, **kwargs):
+        if os.getenv(IDEMPOTENCY_DISABLED_ENV):
+            return function(*args, **kwargs)
+
         payload = kwargs.get(data_keyword_argument)
 
         if payload is None:

aws_lambda_powertools/utilities/idempotency/persistence/dynamodb.py

@@ -62,18 +62,37 @@ def __init__(
             >>>     return {"StatusCode": 200}
         """
 
-        boto_config = boto_config or Config()
-        session = boto3_session or boto3.session.Session()
-        self._ddb_resource = session.resource("dynamodb", config=boto_config)
+        self._boto_config = boto_config or Config()
+        self._boto3_session = boto3_session or boto3.session.Session()
+
+        self._table = None
         self.table_name = table_name
-        self.table = self._ddb_resource.Table(self.table_name)
         self.key_attr = key_attr
         self.expiry_attr = expiry_attr
         self.status_attr = status_attr
         self.data_attr = data_attr
         self.validation_key_attr = validation_key_attr
         super(DynamoDBPersistenceLayer, self).__init__()
 
+    @property
+    def table(self):
+        """
+        Caching property to store boto3 dynamodb Table resource
+
+        """
+        if self._table:
+            return self._table
+        ddb_resource = self._boto3_session.resource("dynamodb", config=self._boto_config)
+        self._table = ddb_resource.Table(self.table_name)
+        return self._table
+
+    @table.setter
+    def table(self, table):
+        """
+        Allow table instance variable to be set directly, primarily for use in tests
+        """
+        self._table = table
+
     def _item_to_data_record(self, item: Dict[str, Any]) -> DataRecord:
         """
         Translate raw item records from DynamoDB to DataRecord
@@ -125,7 +144,7 @@ def _put_record(self, data_record: DataRecord) -> None:
                 ExpressionAttributeNames={"#id": self.key_attr, "#now": self.expiry_attr},
                 ExpressionAttributeValues={":now": int(now.timestamp())},
             )
-        except self._ddb_resource.meta.client.exceptions.ConditionalCheckFailedException:
+        except self.table.meta.client.exceptions.ConditionalCheckFailedException:
             logger.debug(f"Failed to put record for already existing idempotency key: {data_record.idempotency_key}")
             raise IdempotencyItemAlreadyExistsError
 

docs/utilities/idempotency.md

@@ -765,6 +765,123 @@ The idempotency utility can be used with the `validator` decorator. Ensure that
 !!! tip "JMESPath Powertools functions are also available"
     Built-in functions known in the validation utility like `powertools_json`, `powertools_base64`, `powertools_base64_gzip` are also available to use in this utility.
 
+
+## Testing your code
+
+The idempotency utility provides several routes to test your code.
+
+### Disabling the idempotency utility
+When testing your code, you may wish to disable the idempotency logic altogether and focus on testing your business logic. To do this, you can set the environment variable `POWERTOOLS_IDEMPOTENCY_DISABLED`
+with a truthy value. If you prefer setting this for specific tests, and are using Pytest, you can use [monkeypatch](https://docs.pytest.org/en/latest/monkeypatch.html) fixture:
+
+=== "tests.py"
+
+    ```python hl_lines="2 3"
+    def test_idempotent_lambda_handler(monkeypatch):
+        # Set POWERTOOLS_IDEMPOTENCY_DISABLED before calling decorated functions
+        monkeypatch.setenv("POWERTOOLS_IDEMPOTENCY_DISABLED", 1)
+
+        result = handler()
+        ...
+    ```
+=== "app.py"
+
+    ```python
+    from aws_lambda_powertools.utilities.idempotency import (
+    DynamoDBPersistenceLayer, idempotent
+    )
+
+    persistence_layer = DynamoDBPersistenceLayer(table_name="idempotency")
+
+    @idempotent(persistence_store=persistence_layer)
+    def handler(event, context):
+        print('expensive operation')
+        return {
+            "payment_id": 12345,
+            "message": "success",
+            "statusCode": 200,
+        }
+    ```
+
+### Testing with DynamoDB Local
+
+To test with [DynamoDB Local](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/DynamoDBLocal.DownloadingAndRunning.html), you can replace the `Table` resource used by the persistence layer with one you create inside your tests. This allows you to set the endpoint_url.
+
+=== "tests.py"
+
+    ```python hl_lines="6 7 8"
+    import boto3
+
+    import app
+
+    def test_idempotent_lambda():
+        # Create our own Table resource using the endpoint for our DynamoDB Local instance
+        resource = boto3.resource("dynamodb", endpoint_url='http://localhost:8000')
+        table = resource.Table(app.persistence_layer.table_name)
+        app.persistence_layer.table = table
+
+        result = app.handler({'testkey': 'testvalue'}, {})
+        assert result['payment_id'] == 12345
+    ```
+
+=== "app.py"
+
+    ```python
+    from aws_lambda_powertools.utilities.idempotency import (
+    DynamoDBPersistenceLayer, idempotent
+    )
+
+    persistence_layer = DynamoDBPersistenceLayer(table_name="idempotency")
+
+    @idempotent(persistence_store=persistence_layer)
+    def handler(event, context):
+        print('expensive operation')
+        return {
+            "payment_id": 12345,
+            "message": "success",
+            "statusCode": 200,
+        }
+    ```
+
+### How do I mock all DynamoDB I/O operations
+
+The idempotency utility lazily creates the dynamodb [Table](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/dynamodb.html#table) which it uses to access DynamoDB.
+This means it is possible to pass a mocked Table resource, or stub various methods.
+
+=== "tests.py"
+
+    ```python hl_lines="6 7 8 9"
+    from unittest.mock import MagicMock
+
+    import app
+
+    def test_idempotent_lambda():
+        table = MagicMock()
+        app.persistence_layer.table = table
+        result = app.handler({'testkey': 'testvalue'}, {})
+        table.put_item.assert_called()
+        ...
+    ```
+
+=== "app.py"
+
+    ```python
+    from aws_lambda_powertools.utilities.idempotency import (
+    DynamoDBPersistenceLayer, idempotent
+    )
+
+    persistence_layer = DynamoDBPersistenceLayer(table_name="idempotency")
+
+    @idempotent(persistence_store=persistence_layer)
+    def handler(event, context):
+        print('expensive operation')
+        return {
+            "payment_id": 12345,
+            "message": "success",
+            "statusCode": 200,
+        }
+    ```
+
 ## Extra resources
 
 If you're interested in a deep dive on how Amazon uses idempotency when building our APIs, check out

tests/functional/idempotency/test_idempotency.py

@@ -3,6 +3,7 @@
 import json
 import sys
 from hashlib import md5
+from unittest.mock import MagicMock
 
 import jmespath
 import pytest
@@ -994,3 +995,25 @@ def dummy(payload):
 
     # WHEN
     dummy(payload=data_two)
+
+
+def test_idempotency_disabled_envvar(monkeypatch, lambda_context, persistence_store: DynamoDBPersistenceLayer):
+    # Scenario to validate no requests sent to dynamodb table when 'POWERTOOLS_IDEMPOTENCY_DISABLED' is set
+    mock_event = {"data": "value"}
+
+    persistence_store.table = MagicMock()
+
+    monkeypatch.setenv("POWERTOOLS_IDEMPOTENCY_DISABLED", "1")
+
+    @idempotent_function(data_keyword_argument="data", persistence_store=persistence_store)
+    def dummy(data):
+        return {"message": "hello"}
+
+    @idempotent(persistence_store=persistence_store)
+    def dummy_handler(event, context):
+        return {"message": "hi"}
+
+    dummy(data=mock_event)
+    dummy_handler(mock_event, lambda_context)
+
+    assert len(persistence_store.table.method_calls) == 0