fix(docs): Extract idempotency code examples

Changes:
- Extract code examples
- Run isort and black
- Fix yaml and python syntax
- Add make task

Related to:
- aws-powertools#1064

Author: michaelbrewer

Diff for: Makefile

@@ -90,3 +90,11 @@ changelog:
 
 mypy:
 	poetry run mypy --pretty aws_lambda_powertools
+
+format-examples:
+	poetry run isort docs/examples
+	poetry run black docs/examples/*/*/*.py
+
+lint-examples:
+	poetry run python3 -m py_compile docs/examples/*/*/*.py
+	cfn-lint docs/examples/*/*/*.yml

Diff for: docs/examples/utilities/idempotency/batch_sample.py

@@ -0,0 +1,27 @@
+from aws_lambda_powertools.utilities.batch import BatchProcessor, EventType, batch_processor
+from aws_lambda_powertools.utilities.data_classes.sqs_event import SQSRecord
+from aws_lambda_powertools.utilities.idempotency import DynamoDBPersistenceLayer, IdempotencyConfig, idempotent_function
+
+processor = BatchProcessor(event_type=EventType.SQS)
+dynamodb = DynamoDBPersistenceLayer(table_name="idem")
+config = IdempotencyConfig(
+    event_key_jmespath="messageId",  # see Choosing a payload subset section
+    use_local_cache=True,
+)
+
+
+@idempotent_function(data_keyword_argument="record", config=config, persistence_store=dynamodb)
+def record_handler(record: SQSRecord):
+    return {"message": record["body"]}
+
+
+@idempotent_function(data_keyword_argument="data", config=config, persistence_store=dynamodb)
+def dummy(arg_one, arg_two, data: dict, **kwargs):
+    return {"data": data}
+
+
+@batch_processor(record_handler=record_handler, processor=processor)
+def lambda_handler(event, context):
+    # `data` parameter must be called as a keyword argument to work
+    dummy("hello", "universe", data="test")
+    return processor.response()

Diff for: docs/examples/utilities/idempotency/bring_your_own_persistent_store.py

@@ -0,0 +1,126 @@
+import datetime
+import logging
+from typing import Any, Dict, Optional
+
+import boto3
+from botocore.config import Config
+
+from aws_lambda_powertools.utilities.idempotency import BasePersistenceLayer
+from aws_lambda_powertools.utilities.idempotency.exceptions import (
+    IdempotencyItemAlreadyExistsError,
+    IdempotencyItemNotFoundError,
+)
+from aws_lambda_powertools.utilities.idempotency.persistence.base import DataRecord
+
+logger = logging.getLogger(__name__)
+
+
+class DynamoDBPersistenceLayer(BasePersistenceLayer):
+    def __init__(
+        self,
+        table_name: str,
+        key_attr: str = "id",
+        expiry_attr: str = "expiration",
+        status_attr: str = "status",
+        data_attr: str = "data",
+        validation_key_attr: str = "validation",
+        boto_config: Optional[Config] = None,
+        boto3_session: Optional[boto3.session.Session] = None,
+    ):
+        boto_config = boto_config or Config()
+        session = boto3_session or boto3.session.Session()
+        self._ddb_resource = session.resource("dynamodb", config=boto_config)
+        self.table_name = table_name
+        self.table = self._ddb_resource.Table(self.table_name)
+        self.key_attr = key_attr
+        self.expiry_attr = expiry_attr
+        self.status_attr = status_attr
+        self.data_attr = data_attr
+        self.validation_key_attr = validation_key_attr
+        super(DynamoDBPersistenceLayer, self).__init__()
+
+    def _item_to_data_record(self, item: Dict[str, Any]) -> DataRecord:
+        """
+        Translate raw item records from DynamoDB to DataRecord
+
+        Parameters
+        ----------
+        item: Dict[str, Union[str, int]]
+            Item format from dynamodb response
+
+        Returns
+        -------
+        DataRecord
+            representation of item
+
+        """
+        return DataRecord(
+            idempotency_key=item[self.key_attr],
+            status=item[self.status_attr],
+            expiry_timestamp=item[self.expiry_attr],
+            response_data=item.get(self.data_attr),
+            payload_hash=item.get(self.validation_key_attr),
+        )
+
+    def _get_record(self, idempotency_key) -> DataRecord:
+        response = self.table.get_item(Key={self.key_attr: idempotency_key}, ConsistentRead=True)
+
+        try:
+            item = response["Item"]
+        except KeyError:
+            raise IdempotencyItemNotFoundError
+        return self._item_to_data_record(item)
+
+    def _put_record(self, data_record: DataRecord) -> None:
+        item = {
+            self.key_attr: data_record.idempotency_key,
+            self.expiry_attr: data_record.expiry_timestamp,
+            self.status_attr: data_record.status,
+        }
+
+        if self.payload_validation_enabled:
+            item[self.validation_key_attr] = data_record.payload_hash
+
+        now = datetime.datetime.now()
+        try:
+            logger.debug(f"Putting record for idempotency key: {data_record.idempotency_key}")
+            self.table.put_item(
+                Item=item,
+                ConditionExpression=f"attribute_not_exists({self.key_attr}) OR {self.expiry_attr} < :now",
+                ExpressionAttributeValues={":now": int(now.timestamp())},
+            )
+        except self._ddb_resource.meta.client.exceptions.ConditionalCheckFailedException:
+            logger.debug(f"Failed to put record for already existing idempotency key: {data_record.idempotency_key}")
+            raise IdempotencyItemAlreadyExistsError
+
+    def _update_record(self, data_record: DataRecord):
+        logger.debug(f"Updating record for idempotency key: {data_record.idempotency_key}")
+        update_expression = "SET #response_data = :response_data, #expiry = :expiry, #status = :status"
+        expression_attr_values = {
+            ":expiry": data_record.expiry_timestamp,
+            ":response_data": data_record.response_data,
+            ":status": data_record.status,
+        }
+        expression_attr_names = {
+            "#response_data": self.data_attr,
+            "#expiry": self.expiry_attr,
+            "#status": self.status_attr,
+        }
+
+        if self.payload_validation_enabled:
+            update_expression += ", #validation_key = :validation_key"
+            expression_attr_values[":validation_key"] = data_record.payload_hash
+            expression_attr_names["#validation_key"] = self.validation_key_attr
+
+        kwargs = {
+            "Key": {self.key_attr: data_record.idempotency_key},
+            "UpdateExpression": update_expression,
+            "ExpressionAttributeValues": expression_attr_values,
+            "ExpressionAttributeNames": expression_attr_names,
+        }
+
+        self.table.update_item(**kwargs)
+
+    def _delete_record(self, data_record: DataRecord) -> None:
+        logger.debug(f"Deleting record for idempotency key: {data_record.idempotency_key}")
+        self.table.delete_item(Key={self.key_attr: data_record.idempotency_key})

Diff for: docs/examples/utilities/idempotency/dataclass_sample.py

@@ -0,0 +1,33 @@
+from dataclasses import dataclass
+
+from aws_lambda_powertools.utilities.idempotency import DynamoDBPersistenceLayer, IdempotencyConfig, idempotent_function
+
+dynamodb = DynamoDBPersistenceLayer(table_name="idem")
+config = IdempotencyConfig(
+    event_key_jmespath="order_id",  # see Choosing a payload subset section
+    use_local_cache=True,
+)
+
+
+@dataclass
+class OrderItem:
+    sku: str
+    description: str
+
+
+@dataclass
+class Order:
+    item: OrderItem
+    order_id: int
+
+
+@idempotent_function(data_keyword_argument="order", config=config, persistence_store=dynamodb)
+def process_order(order: Order):
+    return f"processed order {order.order_id}"
+
+
+order_item = OrderItem(sku="fake", description="sample")
+order = Order(item=order_item, order_id="fake-id")
+
+# `order` parameter must be called as a keyword argument to work
+process_order(order=order)

Diff for: docs/examples/utilities/idempotency/dynamodb_persistence_layer_customization.py

@@ -0,0 +1,10 @@
+from aws_lambda_powertools.utilities.idempotency import DynamoDBPersistenceLayer
+
+persistence_layer = DynamoDBPersistenceLayer(
+    table_name="IdempotencyTable",
+    key_attr="idempotency_key",
+    expiry_attr="expires_at",
+    status_attr="current_status",
+    data_attr="result_data",
+    validation_key_attr="validation_key",
+)

Diff for: docs/examples/utilities/idempotency/idempotency_cache_ttl.py

@@ -0,0 +1,12 @@
+from aws_lambda_powertools.utilities.idempotency import DynamoDBPersistenceLayer, IdempotencyConfig, idempotent
+
+persistence_layer = DynamoDBPersistenceLayer(table_name="IdempotencyTable")
+config = IdempotencyConfig(
+    event_key_jmespath="body",
+    expires_after_seconds=5 * 60,  # 5 minutes
+)
+
+
+@idempotent(config=config, persistence_store=persistence_layer)
+def handler(event, context):
+    ...

Diff for: docs/examples/utilities/idempotency/idempotency_composite_primary_key.py

@@ -0,0 +1,11 @@
+from aws_lambda_powertools.utilities.idempotency import DynamoDBPersistenceLayer, idempotent
+
+persistence_layer = DynamoDBPersistenceLayer(
+    table_name="IdempotencyTable",
+    sort_key_attr="sort_key",
+)
+
+
+@idempotent(persistence_store=persistence_layer)
+def handler(event, context):
+    return {"message": "success", "id": event["body"]["id"]}

Diff for: docs/examples/utilities/idempotency/idempotency_custom_config.py

@@ -0,0 +1,15 @@
+from botocore.config import Config
+
+from aws_lambda_powertools.utilities.idempotency import DynamoDBPersistenceLayer, IdempotencyConfig, idempotent
+
+config = IdempotencyConfig(event_key_jmespath="body")
+boto_config = Config()
+persistence_layer = DynamoDBPersistenceLayer(
+    table_name="IdempotencyTable",
+    boto_config=boto_config,
+)
+
+
+@idempotent(config=config, persistence_store=persistence_layer)
+def handler(event, context):
+    ...

Diff for: docs/examples/utilities/idempotency/idempotency_custom_session.py

@@ -0,0 +1,16 @@
+import boto3
+
+from aws_lambda_powertools.utilities.idempotency import DynamoDBPersistenceLayer, IdempotencyConfig, idempotent
+
+boto3_session = boto3.session.Session()
+persistence_layer = DynamoDBPersistenceLayer(
+    table_name="IdempotencyTable",
+    boto3_session=boto3_session,
+)
+
+config = IdempotencyConfig(event_key_jmespath="body")
+
+
+@idempotent(config=config, persistence_store=persistence_layer)
+def handler(event, context):
+    ...

Diff for: docs/examples/utilities/idempotency/idempotency_exception_sample.py

@@ -0,0 +1,24 @@
+import requests
+
+from aws_lambda_powertools.utilities.idempotency import DynamoDBPersistenceLayer, IdempotencyConfig, idempotent_function
+
+dynamodb = DynamoDBPersistenceLayer(table_name="idem")
+config = IdempotencyConfig(event_key_jmespath="order_id")
+
+
+def lambda_handler(event, context):
+    # If an exception is raised here, no idempotent record will ever get created as the
+    # idempotent function does not get called
+    do_some_stuff()
+
+    result = call_external_service(data={"user": "user1", "id": 5})
+
+    # This exception will not cause the idempotent record to be deleted, since it
+    # happens after the decorated function has been successfully called
+    raise Exception
+
+
+@idempotent_function(data_keyword_argument="data", config=config, persistence_store=dynamodb)
+def call_external_service(data: dict, **kwargs):
+    result = requests.post("http://example.com", json={"user": data["user"], "transaction_id": data["id"]})
+    return result.json()

Diff for: docs/examples/utilities/idempotency/idempotency_in_memory_cache.py

@@ -0,0 +1,12 @@
+from aws_lambda_powertools.utilities.idempotency import DynamoDBPersistenceLayer, IdempotencyConfig, idempotent
+
+persistence_layer = DynamoDBPersistenceLayer(table_name="IdempotencyTable")
+config = IdempotencyConfig(
+    event_key_jmespath="body",
+    use_local_cache=True,
+)
+
+
+@idempotent(config=config, persistence_store=persistence_layer)
+def handler(event, context):
+    ...

Diff for: docs/examples/utilities/idempotency/idempotency_key_required.py

@@ -0,0 +1,14 @@
+from aws_lambda_powertools.utilities.idempotency import DynamoDBPersistenceLayer, IdempotencyConfig, idempotent
+
+persistence_layer = DynamoDBPersistenceLayer(table_name="IdempotencyTable")
+
+# Requires "user"."uid" and "order_id" to be present
+config = IdempotencyConfig(
+    event_key_jmespath="[user.uid, order_id]",
+    raise_on_no_idempotency_key=True,
+)
+
+
+@idempotent(config=config, persistence_store=persistence_layer)
+def handler(event, context):
+    ...

Diff for: docs/examples/utilities/idempotency/idempotency_payload_validation.py

@@ -0,0 +1,25 @@
+from aws_lambda_powertools.utilities.idempotency import DynamoDBPersistenceLayer, IdempotencyConfig, idempotent
+
+config = IdempotencyConfig(
+    event_key_jmespath="[userDetail, productId]",
+    payload_validation_jmespath="amount",
+)
+persistence_layer = DynamoDBPersistenceLayer(table_name="IdempotencyTable")
+
+
+@idempotent(config=config, persistence_store=persistence_layer)
+def handler(event, context):
+    # Creating a subscription payment is a side
+    # effect of calling this function!
+    payment = create_subscription_payment(
+        user=event["userDetail"]["username"],
+        product=event["product_id"],
+        amount=event["amount"],
+    )
+    ...
+    return {
+        "message": "success",
+        "statusCode": 200,
+        "payment_id": payment.id,
+        "amount": payment.amount,
+    }

Diff for: docs/examples/utilities/idempotency/idempotency_with_validator.py

@@ -0,0 +1,12 @@
+from aws_lambda_powertools.utilities.idempotency import DynamoDBPersistenceLayer, IdempotencyConfig, idempotent
+from aws_lambda_powertools.utilities.validation import envelopes, validator
+
+config = IdempotencyConfig(event_key_jmespath="[message, username]")
+persistence_layer = DynamoDBPersistenceLayer(table_name="IdempotencyTable")
+
+
+@validator(envelope=envelopes.API_GATEWAY_HTTP)
+@idempotent(config=config, persistence_store=persistence_layer)
+def lambda_handler(event, context):
+    cause_some_side_effects(event["username"])
+    return {"message": event["message"], "statusCode": 200}

Diff for: docs/examples/utilities/idempotency/idempotent_decorator.py

@@ -0,0 +1,14 @@
+from aws_lambda_powertools.utilities.idempotency import DynamoDBPersistenceLayer, idempotent
+
+persistence_layer = DynamoDBPersistenceLayer(table_name="IdempotencyTable")
+
+
+@idempotent(persistence_store=persistence_layer)
+def handler(event, context):
+    payment = create_subscription_payment(user=event["user"], product=event["product_id"])
+    ...
+    return {
+        "payment_id": payment.id,
+        "message": "success",
+        "statusCode": 200,
+    }