From b66fa8404a75a6a91ed7e5ca2e4c77ec16a15a8d Mon Sep 17 00:00:00 2001 From: Alexander Melnyk Date: Wed, 13 Sep 2023 18:37:38 +0200 Subject: [PATCH 1/3] add batch integration to idempotency docs --- docs/snippets/idempotency/workingWithBatch.ts | 44 +++ docs/utilities/idempotency.md | 288 +++++++++++++----- 2 files changed, 251 insertions(+), 81 deletions(-) create mode 100644 docs/snippets/idempotency/workingWithBatch.ts diff --git a/docs/snippets/idempotency/workingWithBatch.ts b/docs/snippets/idempotency/workingWithBatch.ts new file mode 100644 index 0000000000..5003c6bd16 --- /dev/null +++ b/docs/snippets/idempotency/workingWithBatch.ts @@ -0,0 +1,44 @@ +import { + BatchProcessor, + EventType, + processPartialResponse, +} from '@aws-lambda-powertools/batch'; + +import { Logger } from '@aws-lambda-powertools/logger'; +import { Context, SQSBatchResponse, SQSEvent, SQSRecord } from 'aws-lambda'; +import { DynamoDBPersistenceLayer } from '@aws-lambda-powertools/idempotency/lib/persistence/DynamoDBPersistenceLayer'; +import { + IdempotencyConfig, + makeIdempotent, +} from '@aws-lambda-powertools/idempotency'; + +const logger = new Logger(); + +const processor = new BatchProcessor(EventType.SQS); + +const dynamoDBPersistence = new DynamoDBPersistenceLayer({ + tableName: 'idempotencyTable', +}); +const idempotencyConfig = new IdempotencyConfig({ + eventKeyJmesPath: 'messageId', +}); + +const processIdempotently = makeIdempotent( + async (record: SQSRecord) => { + logger.info('Processing event', { record }); + // process your event + }, + { + persistenceStore: dynamoDBPersistence, + config: idempotencyConfig, + } +); + +export const handler = async ( + event: SQSEvent, + context: Context +): Promise => { + return processPartialResponse(event, processIdempotently, processor, { + context, + }); +}; diff --git a/docs/utilities/idempotency.md b/docs/utilities/idempotency.md index 5b2cc4fb13..fbdb52d2c6 100644 --- a/docs/utilities/idempotency.md +++ b/docs/utilities/idempotency.md @@ -4,9 +4,14 @@ description: Utility --- ???+ warning - **This utility is currently released as beta developer preview** and is intended strictly for feedback and testing purposes **and not for production workloads**. The version and all future versions tagged with the `-beta` suffix should be treated as not stable. Up until before the [General Availability release](https://github.com/aws-powertools/powertools-lambda-typescript/milestone/7) we might introduce significant breaking changes and improvements in response to customers feedback. +**This utility is currently released as beta developer preview** and is intended strictly for feedback and testing +purposes **and not for production workloads**. The version and all future versions tagged with the `-beta` suffix should +be treated as not stable. Up until before +the [General Availability release](https://github.com/aws-powertools/powertools-lambda-typescript/milestone/7) we might +introduce significant breaking changes and improvements in response to customers feedback. -The idempotency utility provides a simple solution to convert your Lambda functions into idempotent operations which are safe to retry. +The idempotency utility provides a simple solution to convert your Lambda functions into idempotent operations which are +safe to retry. ## Key features @@ -18,13 +23,17 @@ The idempotency utility provides a simple solution to convert your Lambda functi ## Terminology -The property of idempotency means that an operation does not cause additional side effects if it is called more than once with the same input parameters. +The property of idempotency means that an operation does not cause additional side effects if it is called more than +once with the same input parameters. -**Idempotent operations will return the same result when they are called multiple times with the same parameters**. This makes idempotent operations safe to retry. +**Idempotent operations will return the same result when they are called multiple times with the same parameters**. This +makes idempotent operations safe to retry. -**Idempotency key** is a hash representation of either the entire event or a specific configured subset of the event, and invocation results are **JSON serialized** and stored in your persistence storage layer. +**Idempotency key** is a hash representation of either the entire event or a specific configured subset of the event, +and invocation results are **JSON serialized** and stored in your persistence storage layer. -**Idempotency record** is the data representation of an idempotent request saved in your preferred storage layer. We use it to coordinate whether a request is idempotent, whether it's still valid or expired based on timestamps, etc. +**Idempotency record** is the data representation of an idempotent request saved in your preferred storage layer. We use +it to coordinate whether a request is idempotent, whether it's still valid or expired based on timestamps, etc.
```mermaid @@ -53,39 +62,48 @@ classDiagram ## Getting started ### Installation + Install the library in your project + ```shell npm i @aws-lambda-powertools/idempotency @aws-sdk/client-dynamodb ``` -While we support Amazon DynamoDB as a persistance layer out of the box, you need to bring your own AWS SDK for JavaScript v3 DynamoDB client. - +While we support Amazon DynamoDB as a persistance layer out of the box, you need to bring your own AWS SDK for +JavaScript v3 DynamoDB client. ???+ note - This utility supports **[AWS SDK for JavaScript v3](https://docs.aws.amazon.com/AWSJavaScriptSDK/v3/latest/){target="_blank"} only**. If you are using the `nodejs18.x` runtime, the AWS SDK for JavaScript v3 is already installed and you can install only the utility. - +This utility supports **[AWS SDK for JavaScript v3](https://docs.aws.amazon.com/AWSJavaScriptSDK/v3/latest/){target=" +_blank"} only**. If you are using the `nodejs18.x` runtime, the AWS SDK for JavaScript v3 is already installed and you +can install only the utility. ### IAM Permissions -Your Lambda function IAM Role must have `dynamodb:GetItem`, `dynamodb:PutItem`, `dynamodb:UpdateItem` and `dynamodb:DeleteItem` IAM permissions before using this feature. If you're using one of our examples: [AWS Serverless Application Model (SAM)](#required-resources) or [Terraform](#required-resources) the required permissions are already included. +Your Lambda function IAM Role must have `dynamodb:GetItem`, `dynamodb:PutItem`, `dynamodb:UpdateItem` +and `dynamodb:DeleteItem` IAM permissions before using this feature. If you're using one of our +examples: [AWS Serverless Application Model (SAM)](#required-resources) or [Terraform](#required-resources) the required +permissions are already included. ### Required resources -Before getting started, you need to create a persistent storage layer where the idempotency utility can store its state - your lambda functions will need read and write access to it. +Before getting started, you need to create a persistent storage layer where the idempotency utility can store its +state - your lambda functions will need read and write access to it. As of now, Amazon DynamoDB is the only supported persistent storage layer, so you'll need to create a table first. **Default table configuration** -If you're not [changing the default configuration for the DynamoDB persistence layer](#dynamodbpersistencelayer), this is the expected default configuration: +If you're not [changing the default configuration for the DynamoDB persistence layer](#dynamodbpersistencelayer), this +is the expected default configuration: -| Configuration | Default value | Notes | -| ------------------ |:--------------|-----------------------------------------------------------------------------------------| -| Partition key | `id` | The id of each idempotency record which a combination of `functionName#hashOfPayload`. | -| TTL attribute name | `expiration` | This can only be configured after your table is created if you're using AWS Console. | +| Configuration | Default value | Notes | +|--------------------|:--------------|----------------------------------------------------------------------------------------| +| Partition key | `id` | The id of each idempotency record which a combination of `functionName#hashOfPayload`. | +| TTL attribute name | `expiration` | This can only be configured after your table is created if you're using AWS Console. | ???+ tip "Tip: You can share a single state table for all functions" - You can reuse the same DynamoDB table to store idempotency state. We add the Lambda function name in addition to the idempotency key as a hash key. +You can reuse the same DynamoDB table to store idempotency state. We add the Lambda function name in addition to the +idempotency key as a hash key. === "AWS Serverless Application Model (SAM) example" @@ -208,19 +226,23 @@ If you're not [changing the default configuration for the DynamoDB persistence l ``` ???+ warning "Warning: Large responses with DynamoDB persistence layer" - When using this utility with DynamoDB, your function's responses must be [smaller than 400KB](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Limits.html#limits-items){target="_blank"}. +When using this utility with DynamoDB, your function's responses must +be [smaller than 400KB](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Limits.html#limits-items) +{target="_blank"}. Larger items cannot be written to DynamoDB and will cause exceptions. ???+ info "Info: DynamoDB" - Each function invocation will generally make 2 requests to DynamoDB. If the - result returned by your Lambda is less than 1kb, you can expect 2 WCUs per invocation. For retried invocations, you will - see 1WCU and 1RCU. Review the [DynamoDB pricing documentation](https://aws.amazon.com/dynamodb/pricing/){target="_blank"} to - estimate the cost. +Each function invocation will generally make 2 requests to DynamoDB. If the +result returned by your Lambda is less than 1kb, you can expect 2 WCUs per invocation. For retried invocations, you will +see 1WCU and 1RCU. Review the [DynamoDB pricing documentation](https://aws.amazon.com/dynamodb/pricing/){target="_ +blank"} to +estimate the cost. ### MakeIdempotent function wrapper -You can quickly start by initializing the `DynamoDBPersistenceLayer` class and using it with the `makeIdempotent` function wrapper on your Lambda handler. +You can quickly start by initializing the `DynamoDBPersistenceLayer` class and using it with the `makeIdempotent` +function wrapper on your Lambda handler. === "index.ts" @@ -234,21 +256,26 @@ You can quickly start by initializing the `DynamoDBPersistenceLayer` class and u --8<-- "docs/snippets/idempotency/types.ts::13" ``` -After processing this request successfully, a second request containing the exact same payload above will now return the same response, ensuring our customer isn't charged twice. - +After processing this request successfully, a second request containing the exact same payload above will now return the +same response, ensuring our customer isn't charged twice. ???+ note - In this example, the entire Lambda handler is treated as a single idempotent operation. If your Lambda handler can cause multiple side effects, or you're only interested in making a specific logic idempotent, use the `makeIdempotent` high-order function only on the function that needs to be idempotent. - - See [Choosing a payload subset for idempotency](#choosing-a-payload-subset-for-idempotency) for more elaborate use cases. +In this example, the entire Lambda handler is treated as a single idempotent operation. If your Lambda handler can cause +multiple side effects, or you're only interested in making a specific logic idempotent, use the `makeIdempotent` +high-order function only on the function that needs to be idempotent. + See [Choosing a payload subset for idempotency](#choosing-a-payload-subset-for-idempotency) for more elaborate use cases. -You can also use the `makeIdempotent` function wrapper on any function that returns a response to make it idempotent. This is useful when you want to make a specific logic idempotent, for example when your Lambda handler performs multiple side effects and you only want to make a specific one idempotent. +You can also use the `makeIdempotent` function wrapper on any function that returns a response to make it idempotent. +This is useful when you want to make a specific logic idempotent, for example when your Lambda handler performs multiple +side effects and you only want to make a specific one idempotent. ???+ warning "Limitation" - Make sure to return a JSON serializable response from your function, otherwise you'll get an error. +Make sure to return a JSON serializable response from your function, otherwise you'll get an error. -When using `makeIdempotent` on arbitrary functions, you can tell us which argument in your function signature has the data we should use via **`dataIndexArgument`**. If you don't specify this argument, we'll use the first argument in the function signature. +When using `makeIdempotent` on arbitrary functions, you can tell us which argument in your function signature has the +data we should use via **`dataIndexArgument`**. If you don't specify this argument, we'll use the first argument in the +function signature. === "index.ts" @@ -262,16 +289,24 @@ When using `makeIdempotent` on arbitrary functions, you can tell us which argume --8<-- "docs/snippets/idempotency/types.ts::13" ``` -The function this example has two arguments, note that while wrapping it with the `makeIdempotent` high-order function, we specify the `dataIndexArgument` as `1` to tell the decorator that the second argument is the one that contains the data we should use to make the function idempotent. Remember that arguments are zero-indexed, so the first argument is `0`, the second is `1`, and so on. - +The function this example has two arguments, note that while wrapping it with the `makeIdempotent` high-order function, +we specify the `dataIndexArgument` as `1` to tell the decorator that the second argument is the one that contains the +data we should use to make the function idempotent. Remember that arguments are zero-indexed, so the first argument +is `0`, the second is `1`, and so on. ### MakeHandlerIdempotent Middy middleware !!! tip "A note about Middy" - Currently we support only Middy `v3.x` that you can install it by running `npm i @middy/core@~3`. - Check their docs to learn more about [Middy and its middleware stack](https://middy.js.org/docs/intro/getting-started){target="_blank"} as well as [best practices when working with Powertools](https://middy.js.org/docs/integrations/lambda-powertools#best-practices){target="_blank"}. +Currently we support only Middy `v3.x` that you can install it by running `npm i @middy/core@~3`. +Check their docs to learn more about [Middy and its middleware stack](https://middy.js.org/docs/intro/getting-started) +{target="_blank"} as well +as [best practices when working with Powertools](https://middy.js.org/docs/integrations/lambda-powertools#best-practices) +{target="_blank"}. -If you are using [Middy](https://middy.js.org){target="_blank"} as your middleware engine, you can use the `makeHandlerIdempotent` middleware to make your Lambda handler idempotent. Similar to the `makeIdempotent` function wrapper, you can quickly make your Lambda handler idempotent by initializing the `DynamoDBPersistenceLayer` class and using it with the `makeHandlerIdempotent` middleware. +If you are using [Middy](https://middy.js.org){target="_blank"} as your middleware engine, you can use +the `makeHandlerIdempotent` middleware to make your Lambda handler idempotent. Similar to the `makeIdempotent` function +wrapper, you can quickly make your Lambda handler idempotent by initializing the `DynamoDBPersistenceLayer` class and +using it with the `makeHandlerIdempotent` middleware. === "index.ts" @@ -287,18 +322,25 @@ If you are using [Middy](https://middy.js.org){target="_blank"} as your middlewa ### Choosing a payload subset for idempotency -Use [`IdempotencyConfig`](#customizing-the-default-behavior) to instruct the idempotent decorator to only use a portion of your payload to verify whether a request is idempotent, and therefore it should not be retried. When dealing with a more elaborate payload, where parts of the payload always change, you should use the **`eventKeyJmesPath`** parameter. +Use [`IdempotencyConfig`](#customizing-the-default-behavior) to instruct the idempotent decorator to only use a portion +of your payload to verify whether a request is idempotent, and therefore it should not be retried. When dealing with a +more elaborate payload, where parts of the payload always change, you should use the **`eventKeyJmesPath`** parameter. **Payment scenario** -In this example, we have a Lambda handler that creates a payment for a user subscribing to a product. We want to ensure that we don't accidentally charge our customer by subscribing them more than once. +In this example, we have a Lambda handler that creates a payment for a user subscribing to a product. We want to ensure +that we don't accidentally charge our customer by subscribing them more than once. -Imagine the function executes successfully, but the client never receives the response due to a connection issue. It is safe to retry in this instance, as the idempotent decorator will return a previously saved response. +Imagine the function executes successfully, but the client never receives the response due to a connection issue. It is +safe to retry in this instance, as the idempotent decorator will return a previously saved response. -**What we want here** is to instruct Idempotency to use the `user` and `productId` fields from our incoming payload as our idempotency key. If we were to treat the entire request as our idempotency key, a simple HTTP header or timestamp change would cause our customer to be charged twice. +**What we want here** is to instruct Idempotency to use the `user` and `productId` fields from our incoming payload as +our idempotency key. If we were to treat the entire request as our idempotency key, a simple HTTP header or timestamp +change would cause our customer to be charged twice. ???+ warning "Deserializing JSON strings in payloads for increased accuracy." - The payload extracted by the `eventKeyJmesPath` is treated as a string by default. This means there could be differences in whitespace even when the JSON payload itself is identical. +The payload extracted by the `eventKeyJmesPath` is treated as a string by default. This means there could be differences +in whitespace even when the JSON payload itself is identical. === "index.ts" @@ -349,18 +391,23 @@ Imagine the function executes successfully, but the client never receives the re ### Lambda timeouts - - -To prevent against extended failed retries when a [Lambda function times out](https://aws.amazon.com/premiumsupport/knowledge-center/lambda-verify-invocation-timeouts/), Powertools for AWS Lambda calculates and includes the remaining invocation available time as part of the idempotency record. -This is automatically done when you wrap your Lambda handler with the [makeIdempotent](#makeIdempotent-function-wrapper) function wrapper, or use the [`makeHandlerIdempotent`](#makeHandlerIdempotent-middy-middleware) Middy middleware. +To prevent against extended failed retries when +a [Lambda function times out](https://aws.amazon.com/premiumsupport/knowledge-center/lambda-verify-invocation-timeouts/), +Powertools for AWS Lambda calculates and includes the remaining invocation available time as part of the idempotency +record. +This is automatically done when you wrap your Lambda handler with the [makeIdempotent](#makeIdempotent-function-wrapper) +function wrapper, or use the [`makeHandlerIdempotent`](#makeHandlerIdempotent-middy-middleware) Middy middleware. ???+ example - If a second invocation happens **after** this timestamp, and the record is marked as `INPROGRESS`, we will execute the invocation again as if it was in the `EXPIRED` state (e.g, `expire_seconds` field elapsed). +If a second invocation happens **after** this timestamp, and the record is marked as `INPROGRESS`, we will execute the +invocation again as if it was in the `EXPIRED` state (e.g, `expire_seconds` field elapsed). This means that if an invocation expired during execution, it will be quickly executed again on the next retry. ???+ important - If you are only using the [makeIdempotent function wrapper](#makeIdempotent-function-wrapper) to guard isolated parts of your code outside of your handler, you must use `registerLambdaContext` available in the [idempotency config object](#customizing-the-default-behavior) to benefit from this protection. +If you are only using the [makeIdempotent function wrapper](#makeIdempotent-function-wrapper) to guard isolated parts of +your code outside of your handler, you must use `registerLambdaContext` available in +the [idempotency config object](#customizing-the-default-behavior) to benefit from this protection. Here is an example on how you register the Lambda context in your handler: @@ -372,8 +419,10 @@ Here is an example on how you register the Lambda context in your handler: ### Handling exceptions -If you are making on your entire Lambda handler idempotent, any unhandled exceptions that are raised during the code execution will cause **the record in the persistence layer to be deleted**. -This means that new invocations will execute your code again despite having the same payload. If you don't want the record to be deleted, you need to catch exceptions within the idempotent function and return a successful response. +If you are making on your entire Lambda handler idempotent, any unhandled exceptions that are raised during the code +execution will cause **the record in the persistence layer to be deleted**. +This means that new invocations will execute your code again despite having the same payload. If you don't want the +record to be deleted, you need to catch exceptions within the idempotent function and return a successful response.
```mermaid @@ -394,9 +443,13 @@ sequenceDiagram Idempotent sequence exception
-If you are using `makeIdempotent` on any other function, any unhandled exceptions that are thrown _inside_ the wrapped function will cause the record in the persistence layer to be deleted, and allow the function to be executed again if retried. +If you are using `makeIdempotent` on any other function, any unhandled exceptions that are thrown _inside_ the wrapped +function will cause the record in the persistence layer to be deleted, and allow the function to be executed again if +retried. -If an error is thrown _outside_ the scope of the decorated function and after your function has been called, the persistent record will not be affected. In this case, idempotency will be maintained for your decorated function. Example: +If an error is thrown _outside_ the scope of the decorated function and after your function has been called, the +persistent record will not be affected. In this case, idempotency will be maintained for your decorated function. +Example: === "Handling exceptions" @@ -405,7 +458,7 @@ If an error is thrown _outside_ the scope of the decorated function and after yo ``` ???+ warning - **We will throw `IdempotencyPersistenceLayerError`** if any of the calls to the persistence layer fail unexpectedly. +**We will throw `IdempotencyPersistenceLayerError`** if any of the calls to the persistence layer fail unexpectedly. As this happens outside the scope of your decorated function, you are not able to catch it when making your Lambda handler idempotent. @@ -615,7 +668,8 @@ sequenceDiagram #### DynamoDBPersistenceLayer -This persistence layer is built-in, and you can either use an existing DynamoDB table or create a new one dedicated for idempotency state (recommended). +This persistence layer is built-in, and you can either use an existing DynamoDB table or create a new one dedicated for +idempotency state (recommended). === "Customizing DynamoDBPersistenceLayer to suit your table structure" @@ -623,10 +677,11 @@ This persistence layer is built-in, and you can either use an existing DynamoDB --8<-- "docs/snippets/idempotency/customizePersistenceLayer.ts" ``` -When using DynamoDB as a persistence layer, you can alter the attribute names by passing these parameters when initializing the persistence layer: +When using DynamoDB as a persistence layer, you can alter the attribute names by passing these parameters when +initializing the persistence layer: | Parameter | Required | Default | Description | -| ------------------------ | ------------------ | ------------------------------------ | -------------------------------------------------------------------------------------------------------- | +|--------------------------|--------------------|--------------------------------------|----------------------------------------------------------------------------------------------------------| | **tableName** | :heavy_check_mark: | | Table name to store state | | **keyAttr** | | `id` | Partition key of the table. Hashed representation of the payload (unless **sort_key_attr** is specified) | | **expiryAttr** | | `expiration` | Unix timestamp of when record expires | @@ -639,10 +694,11 @@ When using DynamoDB as a persistence layer, you can alter the attribute names by ### Customizing the default behavior -Idempotent decorator can be further configured with **`IdempotencyConfig`** as seen in the previous examples. These are the available options for further configuration +Idempotent decorator can be further configured with **`IdempotencyConfig`** as seen in the previous examples. These are +the available options for further configuration | Parameter | Default | Description | -| ----------------------------- | ------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +|-------------------------------|---------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | **eventKeyJmespath** | `''` | JMESPath expression to extract the idempotency key from the event | | **payloadValidationJmespath** | `''` | JMESPath expression to validate whether certain parameters have changed in the event while the event payload | | **throwOnNoIdempotencyKey** | `false` | Throw an error if no idempotency key was found in the request | @@ -653,19 +709,23 @@ Idempotent decorator can be further configured with **`IdempotencyConfig`** as s ### Handling concurrent executions with the same payload -This utility will throw an **`IdempotencyAlreadyInProgressError`** error if you receive **multiple invocations with the same payload while the first invocation hasn't completed yet**. +This utility will throw an **`IdempotencyAlreadyInProgressError`** error if you receive **multiple invocations with the +same payload while the first invocation hasn't completed yet**. ???+ info - If you receive `IdempotencyAlreadyInProgressError`, you can safely retry the operation. +If you receive `IdempotencyAlreadyInProgressError`, you can safely retry the operation. -This is a locking mechanism for correctness. Since we don't know the result from the first invocation yet, we can't safely allow another concurrent execution. +This is a locking mechanism for correctness. Since we don't know the result from the first invocation yet, we can't +safely allow another concurrent execution. ### Using in-memory cache -**By default, in-memory local caching is disabled**, since we don't know how much memory you consume per invocation compared to the maximum configured in your Lambda function. +**By default, in-memory local caching is disabled**, since we don't know how much memory you consume per invocation +compared to the maximum configured in your Lambda function. ???+ note "Note: This in-memory cache is local to each Lambda execution environment" - This means it will be effective in cases where your function's concurrency is low in comparison to the number of "retry" invocations with the same payload, because cache might be empty. +This means it will be effective in cases where your function's concurrency is low in comparison to the number of "retry" +invocations with the same payload, because cache might be empty. You can enable in-memory caching with the **`useLocalCache`** parameter: @@ -675,13 +735,15 @@ You can enable in-memory caching with the **`useLocalCache`** parameter: --8<-- "docs/snippets/idempotency/workingWithLocalCache.ts" ``` -When enabled, the default is to cache a maximum of 256 records in each Lambda execution environment - You can change it with the **`maxLocalCacheSize`** parameter. +When enabled, the default is to cache a maximum of 256 records in each Lambda execution environment - You can change it +with the **`maxLocalCacheSize`** parameter. ### Expiring idempotency records !!! note "By default, we expire idempotency records after **an hour** (3600 seconds)." -In most cases, it is not desirable to store the idempotency records forever. Rather, you want to guarantee that the same payload won't be executed within a period of time. +In most cases, it is not desirable to store the idempotency records forever. Rather, you want to guarantee that the same +payload won't be executed within a period of time. You can change this window with the **`expiresAfterSeconds`** parameter: @@ -691,10 +753,12 @@ You can change this window with the **`expiresAfterSeconds`** parameter: --8<-- "docs/snippets/idempotency/workingWithRecordExpiration.ts" ``` -This will mark any records older than 5 minutes as expired, and [your function will be executed as normal if it is invoked with a matching payload](#expired-idempotency-records). +This will mark any records older than 5 minutes as expired, +and [your function will be executed as normal if it is invoked with a matching payload](#expired-idempotency-records). ???+ important "Idempotency record expiration vs DynamoDB time-to-live (TTL)" - [DynamoDB TTL is a feature](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/howitworks-ttl.html){target="_blank"} to remove items after a certain period of time, it may occur within 48 hours of expiration. +[DynamoDB TTL is a feature](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/howitworks-ttl.html) +{target="_blank"} to remove items after a certain period of time, it may occur within 48 hours of expiration. We don't rely on DynamoDB or any persistence storage layer to determine whether a record is expired to avoid eventual inconsistency states. @@ -706,12 +770,16 @@ This will mark any records older than 5 minutes as expired, and [your function w ### Payload validation -???+ question "Question: What if your function is invoked with the same payload except some outer parameters have changed?" - Example: A payment transaction for a given productID was requested twice for the same customer, **however the amount to be paid has changed in the second transaction**. +???+ question "Question: What if your function is invoked with the same payload except some outer parameters have +changed?" +Example: A payment transaction for a given productID was requested twice for the same customer, **however the amount to +be paid has changed in the second transaction**. -By default, we will return the same result as it returned before, however in this instance it may be misleading; we provide a fail fast payload validation to address this edge case. +By default, we will return the same result as it returned before, however in this instance it may be misleading; we +provide a fail fast payload validation to address this edge case. -With **`payloadValidationJmesPath`**, you can provide an additional JMESPath expression to specify which part of the event body should be validated against previous idempotent invocations +With **`payloadValidationJmesPath`**, you can provide an additional JMESPath expression to specify which part of the +event body should be validated against previous idempotent invocations === "Payload validation" @@ -719,23 +787,29 @@ With **`payloadValidationJmesPath`**, you can provide an additional JMESPath exp --8<-- "docs/snippets/idempotency/workingWithPayloadValidation.ts" ``` -In this example, the **`userId`** and **`productId`** keys are used as the payload to generate the idempotency key, as per **`eventKeyJmespath`** parameter. +In this example, the **`userId`** and **`productId`** keys are used as the payload to generate the idempotency key, as +per **`eventKeyJmespath`** parameter. ???+ note - If we try to send the same request but with a different amount, we will raise **`IdempotencyValidationError`**. +If we try to send the same request but with a different amount, we will raise **`IdempotencyValidationError`**. -Without payload validation, we would have returned the same result as we did for the initial request. Since we're also returning an amount in the response, this could be quite confusing for the client. +Without payload validation, we would have returned the same result as we did for the initial request. Since we're also +returning an amount in the response, this could be quite confusing for the client. -By using **`payloadValidationJmesPath="amount"`**, we prevent this potentially confusing behavior and instead throw an error. +By using **`payloadValidationJmesPath="amount"`**, we prevent this potentially confusing behavior and instead throw an +error. ### Making idempotency key required If you want to enforce that an idempotency key is required, you can set **`throwOnNoIdempotencyKey`** to `true`. -This means that we will raise **`IdempotencyKeyError`** if the evaluation of **`eventKeyJmesPath`** results in an empty subset. +This means that we will raise **`IdempotencyKeyError`** if the evaluation of **`eventKeyJmesPath`** results in an empty +subset. ???+ warning - To prevent errors, transactions will not be treated as idempotent if **`throwOnNoIdempotencyKey`** is set to `false` and the evaluation of **`eventKeyJmesPath`** is an empty result. Therefore, no data will be fetched, stored, or deleted in the idempotency storage layer. +To prevent errors, transactions will not be treated as idempotent if **`throwOnNoIdempotencyKey`** is set to `false` and +the evaluation of **`eventKeyJmesPath`** is an empty result. Therefore, no data will be fetched, stored, or deleted in +the idempotency storage layer. === "Idempotency key required" @@ -767,9 +841,59 @@ This means that we will raise **`IdempotencyKeyError`** if the evaluation of **` } ``` +### Batch integration + +You can easily integrate with [Batch](batch.md) utility by using idempotency wrapper around your processing function. +This ensures that you process each record in an idempotent manner, and guard against a [Lambda timeout](#lambda-timeouts) idempotent situation. + +???+ "Choosing a unique batch record attribute" + In this example, we choose messageId as our idempotency key since we know it'll be unique. + Depending on your use case, it might be more accurate [to choose another field](#choosing-a-payload-subset-for-idempotency) your producer intentionally set to define uniqueness. + + + +=== "Integration with batch processor" + + ```typescript hl_lines="26-35" + --8<-- "docs/snippets/idempotency/workingWithBatch.ts" + ``` + +=== "Sample event" + + ```json hl_lines="4" + { + "Records": [ + { + "messageId": "059f36b4-87a3-44ab-83d2-661975830a7d", + "receiptHandle": "AQEBwJnKyrHigUMZj6rYigCgxlaS3SLy0a...", + "body": "Test message.", + "attributes": { + "ApproximateReceiveCount": "1", + "SentTimestamp": "1545082649183", + "SenderId": "AIDAIENQZJOLO23YVJ4VO", + "ApproximateFirstReceiveTimestamp": "1545082649185" + }, + "messageAttributes": { + "testAttr": { + "stringValue": "100", + "binaryValue": "base64Str", + "dataType": "Number" + } + }, + "md5OfBody": "e4e68fb7bd0e697a0ae8f1bb342846b3", + "eventSource": "aws:sqs", + "eventSourceARN": "arn:aws:sqs:us-east-2:123456789012:my-queue", + "awsRegion": "us-east-2" + } + ] + } + ``` + ### Customizing AWS SDK configuration -The **`clientConfig`** and **`awsSdkV3Client`** parameters enable you to pass in custom configurations or your own [DynamoDBClient](https://docs.aws.amazon.com/AWSJavaScriptSDK/v3/latest/client/dynamodb/){target="_blank"} when constructing the persistence store. +The **`clientConfig`** and **`awsSdkV3Client`** parameters enable you to pass in custom configurations or your +own [DynamoDBClient](https://docs.aws.amazon.com/AWSJavaScriptSDK/v3/latest/client/dynamodb/){target="_blank"} when +constructing the persistence store. === "Passing specific configuration" @@ -785,9 +909,11 @@ The **`clientConfig`** and **`awsSdkV3Client`** parameters enable you to pass in ### Using a DynamoDB table with a composite primary key -When using a composite primary key table (hash+range key), use `sortKeyAttr` parameter when initializing your persistence layer. +When using a composite primary key table (hash+range key), use `sortKeyAttr` parameter when initializing your +persistence layer. -With this setting, we will save the idempotency key in the sort key instead of the primary key. By default, the primary key will now be set to `idempotency#{LAMBDA_FUNCTION_NAME}`. +With this setting, we will save the idempotency key in the sort key instead of the primary key. By default, the primary +key will now be set to `idempotency#{LAMBDA_FUNCTION_NAME}`. You can optionally set a static value for the partition key using the `staticPkValue` parameter. @@ -800,7 +926,7 @@ You can optionally set a static value for the partition key using the `staticPkV The example function above would cause data to be stored in DynamoDB like this: | id | sort_key | expiration | status | data | -| ---------------------------- | -------------------------------- | ---------- | ----------- | ---------------------------------------------------------------- | +|------------------------------|----------------------------------|------------|-------------|------------------------------------------------------------------| | idempotency#MyLambdaFunction | 1e956ef7da78d0cb890be999aecc0c9e | 1636549553 | COMPLETED | {"paymentId": "12345, "message": "success", "statusCode": 200} | | idempotency#MyLambdaFunction | 2b2cdb5f86361e97b4383087c1ffdf27 | 1636549571 | COMPLETED | {"paymentId": "527212", "message": "success", "statusCode": 200} | | idempotency#MyLambdaFunction | f091d2527ad1c78f05d54cc3f363be80 | 1636549585 | IN_PROGRESS | | From 0d12402051c0666988191f9441d332c98ff544fa Mon Sep 17 00:00:00 2001 From: Alexander Melnyk Date: Wed, 13 Sep 2023 18:54:38 +0200 Subject: [PATCH 2/3] revert unintended IDE formatting changes --- docs/utilities/idempotency.md | 240 ++++++++++++---------------------- 1 file changed, 81 insertions(+), 159 deletions(-) diff --git a/docs/utilities/idempotency.md b/docs/utilities/idempotency.md index fbdb52d2c6..0609ff7394 100644 --- a/docs/utilities/idempotency.md +++ b/docs/utilities/idempotency.md @@ -4,14 +4,9 @@ description: Utility --- ???+ warning -**This utility is currently released as beta developer preview** and is intended strictly for feedback and testing -purposes **and not for production workloads**. The version and all future versions tagged with the `-beta` suffix should -be treated as not stable. Up until before -the [General Availability release](https://github.com/aws-powertools/powertools-lambda-typescript/milestone/7) we might -introduce significant breaking changes and improvements in response to customers feedback. + **This utility is currently released as beta developer preview** and is intended strictly for feedback and testing purposes **and not for production workloads**. The version and all future versions tagged with the `-beta` suffix should be treated as not stable. Up until before the [General Availability release](https://github.com/aws-powertools/powertools-lambda-typescript/milestone/7) we might introduce significant breaking changes and improvements in response to customers feedback. -The idempotency utility provides a simple solution to convert your Lambda functions into idempotent operations which are -safe to retry. +The idempotency utility provides a simple solution to convert your Lambda functions into idempotent operations which are safe to retry. ## Key features @@ -23,17 +18,13 @@ safe to retry. ## Terminology -The property of idempotency means that an operation does not cause additional side effects if it is called more than -once with the same input parameters. +The property of idempotency means that an operation does not cause additional side effects if it is called more than once with the same input parameters. -**Idempotent operations will return the same result when they are called multiple times with the same parameters**. This -makes idempotent operations safe to retry. +**Idempotent operations will return the same result when they are called multiple times with the same parameters**. This makes idempotent operations safe to retry. -**Idempotency key** is a hash representation of either the entire event or a specific configured subset of the event, -and invocation results are **JSON serialized** and stored in your persistence storage layer. +**Idempotency key** is a hash representation of either the entire event or a specific configured subset of the event, and invocation results are **JSON serialized** and stored in your persistence storage layer. -**Idempotency record** is the data representation of an idempotent request saved in your preferred storage layer. We use -it to coordinate whether a request is idempotent, whether it's still valid or expired based on timestamps, etc. +**Idempotency record** is the data representation of an idempotent request saved in your preferred storage layer. We use it to coordinate whether a request is idempotent, whether it's still valid or expired based on timestamps, etc.
```mermaid @@ -62,48 +53,39 @@ classDiagram ## Getting started ### Installation - Install the library in your project - ```shell npm i @aws-lambda-powertools/idempotency @aws-sdk/client-dynamodb ``` -While we support Amazon DynamoDB as a persistance layer out of the box, you need to bring your own AWS SDK for -JavaScript v3 DynamoDB client. +While we support Amazon DynamoDB as a persistance layer out of the box, you need to bring your own AWS SDK for JavaScript v3 DynamoDB client. + ???+ note -This utility supports **[AWS SDK for JavaScript v3](https://docs.aws.amazon.com/AWSJavaScriptSDK/v3/latest/){target=" -_blank"} only**. If you are using the `nodejs18.x` runtime, the AWS SDK for JavaScript v3 is already installed and you -can install only the utility. + This utility supports **[AWS SDK for JavaScript v3](https://docs.aws.amazon.com/AWSJavaScriptSDK/v3/latest/){target="_blank"} only**. If you are using the `nodejs18.x` runtime, the AWS SDK for JavaScript v3 is already installed and you can install only the utility. + ### IAM Permissions -Your Lambda function IAM Role must have `dynamodb:GetItem`, `dynamodb:PutItem`, `dynamodb:UpdateItem` -and `dynamodb:DeleteItem` IAM permissions before using this feature. If you're using one of our -examples: [AWS Serverless Application Model (SAM)](#required-resources) or [Terraform](#required-resources) the required -permissions are already included. +Your Lambda function IAM Role must have `dynamodb:GetItem`, `dynamodb:PutItem`, `dynamodb:UpdateItem` and `dynamodb:DeleteItem` IAM permissions before using this feature. If you're using one of our examples: [AWS Serverless Application Model (SAM)](#required-resources) or [Terraform](#required-resources) the required permissions are already included. ### Required resources -Before getting started, you need to create a persistent storage layer where the idempotency utility can store its -state - your lambda functions will need read and write access to it. +Before getting started, you need to create a persistent storage layer where the idempotency utility can store its state - your lambda functions will need read and write access to it. As of now, Amazon DynamoDB is the only supported persistent storage layer, so you'll need to create a table first. **Default table configuration** -If you're not [changing the default configuration for the DynamoDB persistence layer](#dynamodbpersistencelayer), this -is the expected default configuration: +If you're not [changing the default configuration for the DynamoDB persistence layer](#dynamodbpersistencelayer), this is the expected default configuration: -| Configuration | Default value | Notes | -|--------------------|:--------------|----------------------------------------------------------------------------------------| -| Partition key | `id` | The id of each idempotency record which a combination of `functionName#hashOfPayload`. | -| TTL attribute name | `expiration` | This can only be configured after your table is created if you're using AWS Console. | +| Configuration | Default value | Notes | +| ------------------ |:--------------|-----------------------------------------------------------------------------------------| +| Partition key | `id` | The id of each idempotency record which a combination of `functionName#hashOfPayload`. | +| TTL attribute name | `expiration` | This can only be configured after your table is created if you're using AWS Console. | ???+ tip "Tip: You can share a single state table for all functions" -You can reuse the same DynamoDB table to store idempotency state. We add the Lambda function name in addition to the -idempotency key as a hash key. + You can reuse the same DynamoDB table to store idempotency state. We add the Lambda function name in addition to the idempotency key as a hash key. === "AWS Serverless Application Model (SAM) example" @@ -226,23 +208,19 @@ idempotency key as a hash key. ``` ???+ warning "Warning: Large responses with DynamoDB persistence layer" -When using this utility with DynamoDB, your function's responses must -be [smaller than 400KB](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Limits.html#limits-items) -{target="_blank"}. + When using this utility with DynamoDB, your function's responses must be [smaller than 400KB](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Limits.html#limits-items){target="_blank"}. Larger items cannot be written to DynamoDB and will cause exceptions. ???+ info "Info: DynamoDB" -Each function invocation will generally make 2 requests to DynamoDB. If the -result returned by your Lambda is less than 1kb, you can expect 2 WCUs per invocation. For retried invocations, you will -see 1WCU and 1RCU. Review the [DynamoDB pricing documentation](https://aws.amazon.com/dynamodb/pricing/){target="_ -blank"} to -estimate the cost. + Each function invocation will generally make 2 requests to DynamoDB. If the + result returned by your Lambda is less than 1kb, you can expect 2 WCUs per invocation. For retried invocations, you will + see 1WCU and 1RCU. Review the [DynamoDB pricing documentation](https://aws.amazon.com/dynamodb/pricing/){target="_blank"} to + estimate the cost. ### MakeIdempotent function wrapper -You can quickly start by initializing the `DynamoDBPersistenceLayer` class and using it with the `makeIdempotent` -function wrapper on your Lambda handler. +You can quickly start by initializing the `DynamoDBPersistenceLayer` class and using it with the `makeIdempotent` function wrapper on your Lambda handler. === "index.ts" @@ -256,26 +234,21 @@ function wrapper on your Lambda handler. --8<-- "docs/snippets/idempotency/types.ts::13" ``` -After processing this request successfully, a second request containing the exact same payload above will now return the -same response, ensuring our customer isn't charged twice. +After processing this request successfully, a second request containing the exact same payload above will now return the same response, ensuring our customer isn't charged twice. -???+ note -In this example, the entire Lambda handler is treated as a single idempotent operation. If your Lambda handler can cause -multiple side effects, or you're only interested in making a specific logic idempotent, use the `makeIdempotent` -high-order function only on the function that needs to be idempotent. +???+ note + In this example, the entire Lambda handler is treated as a single idempotent operation. If your Lambda handler can cause multiple side effects, or you're only interested in making a specific logic idempotent, use the `makeIdempotent` high-order function only on the function that needs to be idempotent. + See [Choosing a payload subset for idempotency](#choosing-a-payload-subset-for-idempotency) for more elaborate use cases. -You can also use the `makeIdempotent` function wrapper on any function that returns a response to make it idempotent. -This is useful when you want to make a specific logic idempotent, for example when your Lambda handler performs multiple -side effects and you only want to make a specific one idempotent. + +You can also use the `makeIdempotent` function wrapper on any function that returns a response to make it idempotent. This is useful when you want to make a specific logic idempotent, for example when your Lambda handler performs multiple side effects and you only want to make a specific one idempotent. ???+ warning "Limitation" -Make sure to return a JSON serializable response from your function, otherwise you'll get an error. + Make sure to return a JSON serializable response from your function, otherwise you'll get an error. -When using `makeIdempotent` on arbitrary functions, you can tell us which argument in your function signature has the -data we should use via **`dataIndexArgument`**. If you don't specify this argument, we'll use the first argument in the -function signature. +When using `makeIdempotent` on arbitrary functions, you can tell us which argument in your function signature has the data we should use via **`dataIndexArgument`**. If you don't specify this argument, we'll use the first argument in the function signature. === "index.ts" @@ -289,24 +262,16 @@ function signature. --8<-- "docs/snippets/idempotency/types.ts::13" ``` -The function this example has two arguments, note that while wrapping it with the `makeIdempotent` high-order function, -we specify the `dataIndexArgument` as `1` to tell the decorator that the second argument is the one that contains the -data we should use to make the function idempotent. Remember that arguments are zero-indexed, so the first argument -is `0`, the second is `1`, and so on. +The function this example has two arguments, note that while wrapping it with the `makeIdempotent` high-order function, we specify the `dataIndexArgument` as `1` to tell the decorator that the second argument is the one that contains the data we should use to make the function idempotent. Remember that arguments are zero-indexed, so the first argument is `0`, the second is `1`, and so on. + ### MakeHandlerIdempotent Middy middleware !!! tip "A note about Middy" -Currently we support only Middy `v3.x` that you can install it by running `npm i @middy/core@~3`. -Check their docs to learn more about [Middy and its middleware stack](https://middy.js.org/docs/intro/getting-started) -{target="_blank"} as well -as [best practices when working with Powertools](https://middy.js.org/docs/integrations/lambda-powertools#best-practices) -{target="_blank"}. + Currently we support only Middy `v3.x` that you can install it by running `npm i @middy/core@~3`. + Check their docs to learn more about [Middy and its middleware stack](https://middy.js.org/docs/intro/getting-started){target="_blank"} as well as [best practices when working with Powertools](https://middy.js.org/docs/integrations/lambda-powertools#best-practices){target="_blank"}. -If you are using [Middy](https://middy.js.org){target="_blank"} as your middleware engine, you can use -the `makeHandlerIdempotent` middleware to make your Lambda handler idempotent. Similar to the `makeIdempotent` function -wrapper, you can quickly make your Lambda handler idempotent by initializing the `DynamoDBPersistenceLayer` class and -using it with the `makeHandlerIdempotent` middleware. +If you are using [Middy](https://middy.js.org){target="_blank"} as your middleware engine, you can use the `makeHandlerIdempotent` middleware to make your Lambda handler idempotent. Similar to the `makeIdempotent` function wrapper, you can quickly make your Lambda handler idempotent by initializing the `DynamoDBPersistenceLayer` class and using it with the `makeHandlerIdempotent` middleware. === "index.ts" @@ -322,25 +287,18 @@ using it with the `makeHandlerIdempotent` middleware. ### Choosing a payload subset for idempotency -Use [`IdempotencyConfig`](#customizing-the-default-behavior) to instruct the idempotent decorator to only use a portion -of your payload to verify whether a request is idempotent, and therefore it should not be retried. When dealing with a -more elaborate payload, where parts of the payload always change, you should use the **`eventKeyJmesPath`** parameter. +Use [`IdempotencyConfig`](#customizing-the-default-behavior) to instruct the idempotent decorator to only use a portion of your payload to verify whether a request is idempotent, and therefore it should not be retried. When dealing with a more elaborate payload, where parts of the payload always change, you should use the **`eventKeyJmesPath`** parameter. **Payment scenario** -In this example, we have a Lambda handler that creates a payment for a user subscribing to a product. We want to ensure -that we don't accidentally charge our customer by subscribing them more than once. +In this example, we have a Lambda handler that creates a payment for a user subscribing to a product. We want to ensure that we don't accidentally charge our customer by subscribing them more than once. -Imagine the function executes successfully, but the client never receives the response due to a connection issue. It is -safe to retry in this instance, as the idempotent decorator will return a previously saved response. +Imagine the function executes successfully, but the client never receives the response due to a connection issue. It is safe to retry in this instance, as the idempotent decorator will return a previously saved response. -**What we want here** is to instruct Idempotency to use the `user` and `productId` fields from our incoming payload as -our idempotency key. If we were to treat the entire request as our idempotency key, a simple HTTP header or timestamp -change would cause our customer to be charged twice. +**What we want here** is to instruct Idempotency to use the `user` and `productId` fields from our incoming payload as our idempotency key. If we were to treat the entire request as our idempotency key, a simple HTTP header or timestamp change would cause our customer to be charged twice. ???+ warning "Deserializing JSON strings in payloads for increased accuracy." -The payload extracted by the `eventKeyJmesPath` is treated as a string by default. This means there could be differences -in whitespace even when the JSON payload itself is identical. + The payload extracted by the `eventKeyJmesPath` is treated as a string by default. This means there could be differences in whitespace even when the JSON payload itself is identical. === "index.ts" @@ -391,23 +349,18 @@ in whitespace even when the JSON payload itself is identical. ### Lambda timeouts -To prevent against extended failed retries when -a [Lambda function times out](https://aws.amazon.com/premiumsupport/knowledge-center/lambda-verify-invocation-timeouts/), -Powertools for AWS Lambda calculates and includes the remaining invocation available time as part of the idempotency -record. -This is automatically done when you wrap your Lambda handler with the [makeIdempotent](#makeIdempotent-function-wrapper) -function wrapper, or use the [`makeHandlerIdempotent`](#makeHandlerIdempotent-middy-middleware) Middy middleware. + + +To prevent against extended failed retries when a [Lambda function times out](https://aws.amazon.com/premiumsupport/knowledge-center/lambda-verify-invocation-timeouts/), Powertools for AWS Lambda calculates and includes the remaining invocation available time as part of the idempotency record. +This is automatically done when you wrap your Lambda handler with the [makeIdempotent](#makeIdempotent-function-wrapper) function wrapper, or use the [`makeHandlerIdempotent`](#makeHandlerIdempotent-middy-middleware) Middy middleware. ???+ example -If a second invocation happens **after** this timestamp, and the record is marked as `INPROGRESS`, we will execute the -invocation again as if it was in the `EXPIRED` state (e.g, `expire_seconds` field elapsed). + If a second invocation happens **after** this timestamp, and the record is marked as `INPROGRESS`, we will execute the invocation again as if it was in the `EXPIRED` state (e.g, `expire_seconds` field elapsed). This means that if an invocation expired during execution, it will be quickly executed again on the next retry. ???+ important -If you are only using the [makeIdempotent function wrapper](#makeIdempotent-function-wrapper) to guard isolated parts of -your code outside of your handler, you must use `registerLambdaContext` available in -the [idempotency config object](#customizing-the-default-behavior) to benefit from this protection. + If you are only using the [makeIdempotent function wrapper](#makeIdempotent-function-wrapper) to guard isolated parts of your code outside of your handler, you must use `registerLambdaContext` available in the [idempotency config object](#customizing-the-default-behavior) to benefit from this protection. Here is an example on how you register the Lambda context in your handler: @@ -419,10 +372,8 @@ Here is an example on how you register the Lambda context in your handler: ### Handling exceptions -If you are making on your entire Lambda handler idempotent, any unhandled exceptions that are raised during the code -execution will cause **the record in the persistence layer to be deleted**. -This means that new invocations will execute your code again despite having the same payload. If you don't want the -record to be deleted, you need to catch exceptions within the idempotent function and return a successful response. +If you are making on your entire Lambda handler idempotent, any unhandled exceptions that are raised during the code execution will cause **the record in the persistence layer to be deleted**. +This means that new invocations will execute your code again despite having the same payload. If you don't want the record to be deleted, you need to catch exceptions within the idempotent function and return a successful response.
```mermaid @@ -443,13 +394,9 @@ sequenceDiagram Idempotent sequence exception
-If you are using `makeIdempotent` on any other function, any unhandled exceptions that are thrown _inside_ the wrapped -function will cause the record in the persistence layer to be deleted, and allow the function to be executed again if -retried. +If you are using `makeIdempotent` on any other function, any unhandled exceptions that are thrown _inside_ the wrapped function will cause the record in the persistence layer to be deleted, and allow the function to be executed again if retried. -If an error is thrown _outside_ the scope of the decorated function and after your function has been called, the -persistent record will not be affected. In this case, idempotency will be maintained for your decorated function. -Example: +If an error is thrown _outside_ the scope of the decorated function and after your function has been called, the persistent record will not be affected. In this case, idempotency will be maintained for your decorated function. Example: === "Handling exceptions" @@ -458,7 +405,7 @@ Example: ``` ???+ warning -**We will throw `IdempotencyPersistenceLayerError`** if any of the calls to the persistence layer fail unexpectedly. + **We will throw `IdempotencyPersistenceLayerError`** if any of the calls to the persistence layer fail unexpectedly. As this happens outside the scope of your decorated function, you are not able to catch it when making your Lambda handler idempotent. @@ -668,8 +615,7 @@ sequenceDiagram #### DynamoDBPersistenceLayer -This persistence layer is built-in, and you can either use an existing DynamoDB table or create a new one dedicated for -idempotency state (recommended). +This persistence layer is built-in, and you can either use an existing DynamoDB table or create a new one dedicated for idempotency state (recommended). === "Customizing DynamoDBPersistenceLayer to suit your table structure" @@ -677,11 +623,10 @@ idempotency state (recommended). --8<-- "docs/snippets/idempotency/customizePersistenceLayer.ts" ``` -When using DynamoDB as a persistence layer, you can alter the attribute names by passing these parameters when -initializing the persistence layer: +When using DynamoDB as a persistence layer, you can alter the attribute names by passing these parameters when initializing the persistence layer: | Parameter | Required | Default | Description | -|--------------------------|--------------------|--------------------------------------|----------------------------------------------------------------------------------------------------------| +| ------------------------ | ------------------ | ------------------------------------ | -------------------------------------------------------------------------------------------------------- | | **tableName** | :heavy_check_mark: | | Table name to store state | | **keyAttr** | | `id` | Partition key of the table. Hashed representation of the payload (unless **sort_key_attr** is specified) | | **expiryAttr** | | `expiration` | Unix timestamp of when record expires | @@ -694,11 +639,10 @@ initializing the persistence layer: ### Customizing the default behavior -Idempotent decorator can be further configured with **`IdempotencyConfig`** as seen in the previous examples. These are -the available options for further configuration +Idempotent decorator can be further configured with **`IdempotencyConfig`** as seen in the previous examples. These are the available options for further configuration | Parameter | Default | Description | -|-------------------------------|---------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| ----------------------------- | ------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | | **eventKeyJmespath** | `''` | JMESPath expression to extract the idempotency key from the event | | **payloadValidationJmespath** | `''` | JMESPath expression to validate whether certain parameters have changed in the event while the event payload | | **throwOnNoIdempotencyKey** | `false` | Throw an error if no idempotency key was found in the request | @@ -709,23 +653,19 @@ the available options for further configuration ### Handling concurrent executions with the same payload -This utility will throw an **`IdempotencyAlreadyInProgressError`** error if you receive **multiple invocations with the -same payload while the first invocation hasn't completed yet**. +This utility will throw an **`IdempotencyAlreadyInProgressError`** error if you receive **multiple invocations with the same payload while the first invocation hasn't completed yet**. ???+ info -If you receive `IdempotencyAlreadyInProgressError`, you can safely retry the operation. + If you receive `IdempotencyAlreadyInProgressError`, you can safely retry the operation. -This is a locking mechanism for correctness. Since we don't know the result from the first invocation yet, we can't -safely allow another concurrent execution. +This is a locking mechanism for correctness. Since we don't know the result from the first invocation yet, we can't safely allow another concurrent execution. ### Using in-memory cache -**By default, in-memory local caching is disabled**, since we don't know how much memory you consume per invocation -compared to the maximum configured in your Lambda function. +**By default, in-memory local caching is disabled**, since we don't know how much memory you consume per invocation compared to the maximum configured in your Lambda function. ???+ note "Note: This in-memory cache is local to each Lambda execution environment" -This means it will be effective in cases where your function's concurrency is low in comparison to the number of "retry" -invocations with the same payload, because cache might be empty. + This means it will be effective in cases where your function's concurrency is low in comparison to the number of "retry" invocations with the same payload, because cache might be empty. You can enable in-memory caching with the **`useLocalCache`** parameter: @@ -735,15 +675,13 @@ You can enable in-memory caching with the **`useLocalCache`** parameter: --8<-- "docs/snippets/idempotency/workingWithLocalCache.ts" ``` -When enabled, the default is to cache a maximum of 256 records in each Lambda execution environment - You can change it -with the **`maxLocalCacheSize`** parameter. +When enabled, the default is to cache a maximum of 256 records in each Lambda execution environment - You can change it with the **`maxLocalCacheSize`** parameter. ### Expiring idempotency records !!! note "By default, we expire idempotency records after **an hour** (3600 seconds)." -In most cases, it is not desirable to store the idempotency records forever. Rather, you want to guarantee that the same -payload won't be executed within a period of time. +In most cases, it is not desirable to store the idempotency records forever. Rather, you want to guarantee that the same payload won't be executed within a period of time. You can change this window with the **`expiresAfterSeconds`** parameter: @@ -753,12 +691,10 @@ You can change this window with the **`expiresAfterSeconds`** parameter: --8<-- "docs/snippets/idempotency/workingWithRecordExpiration.ts" ``` -This will mark any records older than 5 minutes as expired, -and [your function will be executed as normal if it is invoked with a matching payload](#expired-idempotency-records). +This will mark any records older than 5 minutes as expired, and [your function will be executed as normal if it is invoked with a matching payload](#expired-idempotency-records). ???+ important "Idempotency record expiration vs DynamoDB time-to-live (TTL)" -[DynamoDB TTL is a feature](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/howitworks-ttl.html) -{target="_blank"} to remove items after a certain period of time, it may occur within 48 hours of expiration. + [DynamoDB TTL is a feature](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/howitworks-ttl.html){target="_blank"} to remove items after a certain period of time, it may occur within 48 hours of expiration. We don't rely on DynamoDB or any persistence storage layer to determine whether a record is expired to avoid eventual inconsistency states. @@ -770,16 +706,12 @@ and [your function will be executed as normal if it is invoked with a matching p ### Payload validation -???+ question "Question: What if your function is invoked with the same payload except some outer parameters have -changed?" -Example: A payment transaction for a given productID was requested twice for the same customer, **however the amount to -be paid has changed in the second transaction**. +???+ question "Question: What if your function is invoked with the same payload except some outer parameters have changed?" + Example: A payment transaction for a given productID was requested twice for the same customer, **however the amount to be paid has changed in the second transaction**. -By default, we will return the same result as it returned before, however in this instance it may be misleading; we -provide a fail fast payload validation to address this edge case. +By default, we will return the same result as it returned before, however in this instance it may be misleading; we provide a fail fast payload validation to address this edge case. -With **`payloadValidationJmesPath`**, you can provide an additional JMESPath expression to specify which part of the -event body should be validated against previous idempotent invocations +With **`payloadValidationJmesPath`**, you can provide an additional JMESPath expression to specify which part of the event body should be validated against previous idempotent invocations === "Payload validation" @@ -787,29 +719,23 @@ event body should be validated against previous idempotent invocations --8<-- "docs/snippets/idempotency/workingWithPayloadValidation.ts" ``` -In this example, the **`userId`** and **`productId`** keys are used as the payload to generate the idempotency key, as -per **`eventKeyJmespath`** parameter. +In this example, the **`userId`** and **`productId`** keys are used as the payload to generate the idempotency key, as per **`eventKeyJmespath`** parameter. ???+ note -If we try to send the same request but with a different amount, we will raise **`IdempotencyValidationError`**. + If we try to send the same request but with a different amount, we will raise **`IdempotencyValidationError`**. -Without payload validation, we would have returned the same result as we did for the initial request. Since we're also -returning an amount in the response, this could be quite confusing for the client. +Without payload validation, we would have returned the same result as we did for the initial request. Since we're also returning an amount in the response, this could be quite confusing for the client. -By using **`payloadValidationJmesPath="amount"`**, we prevent this potentially confusing behavior and instead throw an -error. +By using **`payloadValidationJmesPath="amount"`**, we prevent this potentially confusing behavior and instead throw an error. ### Making idempotency key required If you want to enforce that an idempotency key is required, you can set **`throwOnNoIdempotencyKey`** to `true`. -This means that we will raise **`IdempotencyKeyError`** if the evaluation of **`eventKeyJmesPath`** results in an empty -subset. +This means that we will raise **`IdempotencyKeyError`** if the evaluation of **`eventKeyJmesPath`** results in an empty subset. ???+ warning -To prevent errors, transactions will not be treated as idempotent if **`throwOnNoIdempotencyKey`** is set to `false` and -the evaluation of **`eventKeyJmesPath`** is an empty result. Therefore, no data will be fetched, stored, or deleted in -the idempotency storage layer. + To prevent errors, transactions will not be treated as idempotent if **`throwOnNoIdempotencyKey`** is set to `false` and the evaluation of **`eventKeyJmesPath`** is an empty result. Therefore, no data will be fetched, stored, or deleted in the idempotency storage layer. === "Idempotency key required" @@ -891,9 +817,7 @@ This ensures that you process each record in an idempotent manner, and guard aga ### Customizing AWS SDK configuration -The **`clientConfig`** and **`awsSdkV3Client`** parameters enable you to pass in custom configurations or your -own [DynamoDBClient](https://docs.aws.amazon.com/AWSJavaScriptSDK/v3/latest/client/dynamodb/){target="_blank"} when -constructing the persistence store. +The **`clientConfig`** and **`awsSdkV3Client`** parameters enable you to pass in custom configurations or your own [DynamoDBClient](https://docs.aws.amazon.com/AWSJavaScriptSDK/v3/latest/client/dynamodb/){target="_blank"} when constructing the persistence store. === "Passing specific configuration" @@ -909,11 +833,9 @@ constructing the persistence store. ### Using a DynamoDB table with a composite primary key -When using a composite primary key table (hash+range key), use `sortKeyAttr` parameter when initializing your -persistence layer. +When using a composite primary key table (hash+range key), use `sortKeyAttr` parameter when initializing your persistence layer. -With this setting, we will save the idempotency key in the sort key instead of the primary key. By default, the primary -key will now be set to `idempotency#{LAMBDA_FUNCTION_NAME}`. +With this setting, we will save the idempotency key in the sort key instead of the primary key. By default, the primary key will now be set to `idempotency#{LAMBDA_FUNCTION_NAME}`. You can optionally set a static value for the partition key using the `staticPkValue` parameter. @@ -926,7 +848,7 @@ You can optionally set a static value for the partition key using the `staticPkV The example function above would cause data to be stored in DynamoDB like this: | id | sort_key | expiration | status | data | -|------------------------------|----------------------------------|------------|-------------|------------------------------------------------------------------| +| ---------------------------- | -------------------------------- | ---------- | ----------- | ---------------------------------------------------------------- | | idempotency#MyLambdaFunction | 1e956ef7da78d0cb890be999aecc0c9e | 1636549553 | COMPLETED | {"paymentId": "12345, "message": "success", "statusCode": 200} | | idempotency#MyLambdaFunction | 2b2cdb5f86361e97b4383087c1ffdf27 | 1636549571 | COMPLETED | {"paymentId": "527212", "message": "success", "statusCode": 200} | | idempotency#MyLambdaFunction | f091d2527ad1c78f05d54cc3f363be80 | 1636549585 | IN_PROGRESS | | From 8e895f565d64ab061b785a57345a249540fcb660 Mon Sep 17 00:00:00 2001 From: Andrea Amorosi Date: Thu, 14 Sep 2023 00:43:43 +0200 Subject: [PATCH 3/3] chore: fixed imports / highlight --- docs/snippets/idempotency/workingWithBatch.ts | 18 ++++++++++-------- docs/utilities/idempotency.md | 12 ++++++------ 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/docs/snippets/idempotency/workingWithBatch.ts b/docs/snippets/idempotency/workingWithBatch.ts index 5003c6bd16..274ff6d674 100644 --- a/docs/snippets/idempotency/workingWithBatch.ts +++ b/docs/snippets/idempotency/workingWithBatch.ts @@ -3,17 +3,18 @@ import { EventType, processPartialResponse, } from '@aws-lambda-powertools/batch'; - -import { Logger } from '@aws-lambda-powertools/logger'; -import { Context, SQSBatchResponse, SQSEvent, SQSRecord } from 'aws-lambda'; -import { DynamoDBPersistenceLayer } from '@aws-lambda-powertools/idempotency/lib/persistence/DynamoDBPersistenceLayer'; +import type { + Context, + SQSBatchResponse, + SQSEvent, + SQSRecord, +} from 'aws-lambda'; +import { DynamoDBPersistenceLayer } from '@aws-lambda-powertools/idempotency/dynamodb'; import { IdempotencyConfig, makeIdempotent, } from '@aws-lambda-powertools/idempotency'; -const logger = new Logger(); - const processor = new BatchProcessor(EventType.SQS); const dynamoDBPersistence = new DynamoDBPersistenceLayer({ @@ -24,8 +25,7 @@ const idempotencyConfig = new IdempotencyConfig({ }); const processIdempotently = makeIdempotent( - async (record: SQSRecord) => { - logger.info('Processing event', { record }); + async (_record: SQSRecord) => { // process your event }, { @@ -38,6 +38,8 @@ export const handler = async ( event: SQSEvent, context: Context ): Promise => { + idempotencyConfig.registerLambdaContext(context); + return processPartialResponse(event, processIdempotently, processor, { context, }); diff --git a/docs/utilities/idempotency.md b/docs/utilities/idempotency.md index 0609ff7394..59de2a9850 100644 --- a/docs/utilities/idempotency.md +++ b/docs/utilities/idempotency.md @@ -79,10 +79,10 @@ As of now, Amazon DynamoDB is the only supported persistent storage layer, so yo If you're not [changing the default configuration for the DynamoDB persistence layer](#dynamodbpersistencelayer), this is the expected default configuration: -| Configuration | Default value | Notes | -| ------------------ |:--------------|-----------------------------------------------------------------------------------------| -| Partition key | `id` | The id of each idempotency record which a combination of `functionName#hashOfPayload`. | -| TTL attribute name | `expiration` | This can only be configured after your table is created if you're using AWS Console. | +| Configuration | Default value | Notes | +| ------------------ | :------------ | -------------------------------------------------------------------------------------- | +| Partition key | `id` | The id of each idempotency record which a combination of `functionName#hashOfPayload`. | +| TTL attribute name | `expiration` | This can only be configured after your table is created if you're using AWS Console. | ???+ tip "Tip: You can share a single state table for all functions" You can reuse the same DynamoDB table to store idempotency state. We add the Lambda function name in addition to the idempotency key as a hash key. @@ -773,14 +773,14 @@ You can easily integrate with [Batch](batch.md) utility by using idempotency wra This ensures that you process each record in an idempotent manner, and guard against a [Lambda timeout](#lambda-timeouts) idempotent situation. ???+ "Choosing a unique batch record attribute" - In this example, we choose messageId as our idempotency key since we know it'll be unique. + In this example, we choose `messageId` as our idempotency key since we know it'll be unique. Depending on your use case, it might be more accurate [to choose another field](#choosing-a-payload-subset-for-idempotency) your producer intentionally set to define uniqueness. === "Integration with batch processor" - ```typescript hl_lines="26-35" + ```typescript hl_lines="27 31-34 41" --8<-- "docs/snippets/idempotency/workingWithBatch.ts" ```