Skip to content

Commit b5eb84c

Browse files
authored
docs(idempotency): fix, improve, and increase visibility for batch integration (#1776)
1 parent 5a20da7 commit b5eb84c

File tree

1 file changed

+80
-69
lines changed

1 file changed

+80
-69
lines changed

docs/utilities/idempotency.md

+80-69
Original file line numberDiff line numberDiff line change
@@ -132,75 +132,6 @@ When using `idempotent_function`, you must tell us which keyword parameter in yo
132132

133133
DynamoDB Persistency layer uses a Resource client [which is not thread-safe](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/resources.html?highlight=multithreading#multithreading-or-multiprocessing-with-resources){target="_blank"}.
134134

135-
=== "batch_sample.py"
136-
137-
This example also demonstrates how you can integrate with [Batch utility](batch.md), so you can process each record in an idempotent manner.
138-
139-
```python hl_lines="4-5 16 21 30"
140-
from aws_lambda_powertools.utilities.batch import (BatchProcessor, EventType,
141-
batch_processor)
142-
from aws_lambda_powertools.utilities.data_classes.sqs_event import SQSRecord
143-
from aws_lambda_powertools.utilities.idempotency import (
144-
DynamoDBPersistenceLayer, IdempotencyConfig, idempotent_function)
145-
146-
147-
processor = BatchProcessor(event_type=EventType.SQS)
148-
dynamodb = DynamoDBPersistenceLayer(table_name="idem")
149-
config = IdempotencyConfig(
150-
event_key_jmespath="messageId", # see Choosing a payload subset section
151-
use_local_cache=True,
152-
)
153-
154-
155-
@idempotent_function(data_keyword_argument="record", config=config, persistence_store=dynamodb)
156-
def record_handler(record: SQSRecord):
157-
return {"message": record["body"]}
158-
159-
160-
@idempotent_function(data_keyword_argument="data", config=config, persistence_store=dynamodb)
161-
def dummy(arg_one, arg_two, data: dict, **kwargs):
162-
return {"data": data}
163-
164-
165-
@batch_processor(record_handler=record_handler, processor=processor)
166-
def lambda_handler(event, context):
167-
config.register_lambda_context(context) # see Lambda timeouts section
168-
# `data` parameter must be called as a keyword argument to work
169-
dummy("hello", "universe", data="test")
170-
return processor.response()
171-
```
172-
173-
=== "Batch event"
174-
175-
```json hl_lines="4"
176-
{
177-
"Records": [
178-
{
179-
"messageId": "059f36b4-87a3-44ab-83d2-661975830a7d",
180-
"receiptHandle": "AQEBwJnKyrHigUMZj6rYigCgxlaS3SLy0a...",
181-
"body": "Test message.",
182-
"attributes": {
183-
"ApproximateReceiveCount": "1",
184-
"SentTimestamp": "1545082649183",
185-
"SenderId": "AIDAIENQZJOLO23YVJ4VO",
186-
"ApproximateFirstReceiveTimestamp": "1545082649185"
187-
},
188-
"messageAttributes": {
189-
"testAttr": {
190-
"stringValue": "100",
191-
"binaryValue": "base64Str",
192-
"dataType": "Number"
193-
}
194-
},
195-
"md5OfBody": "e4e68fb7bd0e697a0ae8f1bb342846b3",
196-
"eventSource": "aws:sqs",
197-
"eventSourceARN": "arn:aws:sqs:us-east-2:123456789012:my-queue",
198-
"awsRegion": "us-east-2"
199-
}
200-
]
201-
}
202-
```
203-
204135
=== "dataclass_sample.py"
205136

206137
```python hl_lines="3-4 23 33"
@@ -276,6 +207,82 @@ When using `idempotent_function`, you must tell us which keyword parameter in yo
276207
process_order(order=order)
277208
```
278209

210+
#### Batch integration
211+
212+
You can can easily integrate with [Batch utility](batch.md) via context manager. This ensures that you process each record in an idempotent manner, and guard against a [Lambda timeout](#lambda-timeouts) idempotent situation.
213+
214+
???+ "Choosing an unique batch record attribute"
215+
In this example, we choose `messageId` as our idempotency token since we know it'll be unique.
216+
217+
Depending on your use case, it might be more accurate [to choose another field](#choosing-a-payload-subset-for-idempotency) your producer intentionally set to define uniqueness.
218+
219+
=== "batch_sample.py"
220+
221+
```python hl_lines="3-4 10 15 21 25-26 29 31"
222+
from aws_lambda_powertools.utilities.batch import BatchProcessor, EventType
223+
from aws_lambda_powertools.utilities.data_classes.sqs_event import SQSRecord
224+
from aws_lambda_powertools.utilities.idempotency import (
225+
DynamoDBPersistenceLayer, IdempotencyConfig, idempotent_function)
226+
227+
228+
processor = BatchProcessor(event_type=EventType.SQS)
229+
dynamodb = DynamoDBPersistenceLayer(table_name="idem")
230+
config = IdempotencyConfig(
231+
event_key_jmespath="messageId", # see Choosing a payload subset section
232+
use_local_cache=True,
233+
)
234+
235+
236+
@idempotent_function(data_keyword_argument="record", config=config, persistence_store=dynamodb)
237+
def record_handler(record: SQSRecord):
238+
return {"message": record.body}
239+
240+
241+
def lambda_handler(event, context):
242+
config.register_lambda_context(context) # see Lambda timeouts section
243+
244+
# with Lambda context registered for Idempotency
245+
# we can now kick in the Bach processing logic
246+
batch = event["Records"]
247+
with processor(records=batch, handler=record_handler):
248+
# in case you want to access each record processed by your record_handler
249+
# otherwise ignore the result variable assignment
250+
processed_messages = processor.process()
251+
252+
return processor.response()
253+
```
254+
255+
=== "batch_event.json"
256+
257+
```json hl_lines="4"
258+
{
259+
"Records": [
260+
{
261+
"messageId": "059f36b4-87a3-44ab-83d2-661975830a7d",
262+
"receiptHandle": "AQEBwJnKyrHigUMZj6rYigCgxlaS3SLy0a...",
263+
"body": "Test message.",
264+
"attributes": {
265+
"ApproximateReceiveCount": "1",
266+
"SentTimestamp": "1545082649183",
267+
"SenderId": "AIDAIENQZJOLO23YVJ4VO",
268+
"ApproximateFirstReceiveTimestamp": "1545082649185"
269+
},
270+
"messageAttributes": {
271+
"testAttr": {
272+
"stringValue": "100",
273+
"binaryValue": "base64Str",
274+
"dataType": "Number"
275+
}
276+
},
277+
"md5OfBody": "e4e68fb7bd0e697a0ae8f1bb342846b3",
278+
"eventSource": "aws:sqs",
279+
"eventSourceARN": "arn:aws:sqs:us-east-2:123456789012:my-queue",
280+
"awsRegion": "us-east-2"
281+
}
282+
]
283+
}
284+
```
285+
279286
### Choosing a payload subset for idempotency
280287

281288
???+ tip "Tip: Dealing with always changing payloads"
@@ -982,6 +989,10 @@ class DynamoDBPersistenceLayer(BasePersistenceLayer):
982989

983990
## Compatibility with other utilities
984991

992+
### Batch
993+
994+
See [Batch integration](#batch-integration) above.
995+
985996
### Validation utility
986997

987998
The idempotency utility can be used with the `validator` decorator. Ensure that idempotency is the innermost decorator.

0 commit comments

Comments
 (0)