From 51e9f40f72d2a6920d0ab1eb7715bcb56ca9cc30 Mon Sep 17 00:00:00 2001 From: Alexander Schueren Date: Tue, 13 Aug 2024 09:55:12 +0200 Subject: [PATCH 01/10] add param --- packages/idempotency/src/makeIdempotent.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/idempotency/src/makeIdempotent.ts b/packages/idempotency/src/makeIdempotent.ts index 47aa1f3f7d..251a72a8f8 100644 --- a/packages/idempotency/src/makeIdempotent.ts +++ b/packages/idempotency/src/makeIdempotent.ts @@ -71,6 +71,8 @@ const isOptionsWithDataIndexArgument = ( * return Promise.resolve(); * }; * + * @param fn - the function to make idempotent + * @param options - the options to configure the idempotency behavior * ``` */ function makeIdempotent( From 9995be60fdf7f2988bdc81b4958b159ccc307933 Mon Sep 17 00:00:00 2001 From: Alexander Schueren Date: Tue, 13 Aug 2024 11:32:27 +0200 Subject: [PATCH 02/10] change readme and api docs --- packages/idempotency/README.md | 18 +++++------------- packages/idempotency/src/IdempotencyConfig.ts | 8 ++++---- packages/idempotency/src/IdempotencyHandler.ts | 6 +++--- .../idempotency/src/idempotencyDecorator.ts | 4 +++- .../src/types/IdempotencyOptions.ts | 4 ++-- .../idempotency/src/types/IdempotencyRecord.ts | 6 ++++++ packages/idempotency/src/types/index.ts | 1 + packages/idempotency/typedoc.json | 2 +- 8 files changed, 25 insertions(+), 24 deletions(-) diff --git a/packages/idempotency/README.md b/packages/idempotency/README.md index 8b0f568601..3b26db7e4c 100644 --- a/packages/idempotency/README.md +++ b/packages/idempotency/README.md @@ -5,7 +5,6 @@ Powertools for AWS Lambda (TypeScript) is a developer toolkit to implement Serve You can use the package in both TypeScript and JavaScript code bases. - [Intro](#intro) -- [Key features](#key-features) - [Usage](#usage) - [Function wrapper](#function-wrapper) - [Decorator](#decorator) @@ -27,14 +26,6 @@ You can either use it to wrap a function, decorate a method, or as Middy middlew The current implementation provides a persistence layer for Amazon DynamoDB, which offers a variety of configuration options. You can also bring your own persistence layer by extending the `BasePersistenceLayer` class. -## Key features - -- Prevent Lambda handler from executing more than once on the same event payload during a time window -- Ensure Lambda handler returns the same result when called with the same payload -- Select a subset of the event as the idempotency key using JMESPath expressions -- Set a time window in which records with the same payload should be considered duplicates -- Expires in-progress executions if the Lambda function times out halfway through - ## Usage To get started, install the library by running: @@ -49,7 +40,7 @@ Next, review the IAM permissions attached to your AWS Lambda function and make s You can make any function idempotent, and safe to retry, by wrapping it using the `makeIdempotent` higher-order function. -The function wrapper takes a reference to the function to be made idempotent as first argument, and an object with options as second argument. +The `makeIdempotent` function takes a reference to the function to be made idempotent as first argument, and an object with options as second argument. When you wrap your Lambda handler function, the utility uses the content of the `event` parameter to handle the idempotency logic. @@ -57,7 +48,7 @@ When you wrap your Lambda handler function, the utility uses the content of the import { makeIdempotent } from '@aws-lambda-powertools/idempotency'; import { DynamoDBPersistenceLayer } from '@aws-lambda-powertools/idempotency/dynamodb'; import type { Context, APIGatewayProxyEvent } from 'aws-lambda'; - +`` const persistenceStore = new DynamoDBPersistenceLayer({ tableName: 'idempotencyTableName', }); @@ -255,7 +246,8 @@ The decorator configuration options are identical with the ones of the `makeIdem If instead you use Middy, you can use the `makeHandlerIdempotent` middleware. When using the middleware your Lambda handler becomes idempotent. -By default, the Idempotency utility will use the full event payload to create an hash and determine if a request is idempotent, and therefore it should not be retried. When dealing with a more elaborate payload, where parts of the payload always change you should use the `IdempotencyConfig` object to instruct the utility to only use a portion of your payload. This is useful when dealing with payloads that contain timestamps or request ids. +By default, the Idempotency utility will use the full event payload to create an hash and determine if a request is idempotent, and therefore it should not be retried. +When dealing with a more elaborate payload, where parts of the payload always change you should use the `IdempotencyConfig` object to instruct the utility to only use a portion of your payload. This is useful when dealing with payloads that contain timestamps or request ids. ```ts import { IdempotencyConfig } from '@aws-lambda-powertools/idempotency'; @@ -314,7 +306,7 @@ Help us prioritize upcoming functionalities or utilities by [upvoting existing R ### Becoming a reference customer -Knowing which companies are using this library is important to help prioritize the project internally. If your company is using Powertools for AWS Lambda (TypeScript), you can request to have your name and logo added to the README file by raising a [Support Powertools for AWS Lambda (TypeScript) (become a reference)](https://github.com/aws-powertools/powertools-lambda-typescript/issues/new?assignees=&labels=customer-reference&template=support_powertools.yml&title=%5BSupport+Lambda+Powertools%5D%3A+%3Cyour+organization+name%3E) issue. +Knowing which companies are using this library is important to help prioritize the project internally. If your company is using Powertools for AWS Lambda (TypeScript), you can request to have your name and logo added to the README file by raising a [Support Powertools for AWS Lambda (TypeScript) (become a reference)](https://s12d.com/become-reference-pt-ts) issue. The following companies, among others, use Powertools: diff --git a/packages/idempotency/src/IdempotencyConfig.ts b/packages/idempotency/src/IdempotencyConfig.ts index 0a89a4af4c..043fa828f8 100644 --- a/packages/idempotency/src/IdempotencyConfig.ts +++ b/packages/idempotency/src/IdempotencyConfig.ts @@ -1,8 +1,8 @@ -import { EnvironmentVariablesService } from './config/EnvironmentVariablesService.js'; +import { PowertoolsFunctions } from '@aws-lambda-powertools/jmespath/functions'; +import type { JMESPathParsingOptions } from '@aws-lambda-powertools/jmespath/types'; import type { Context } from 'aws-lambda'; +import { EnvironmentVariablesService } from './config/EnvironmentVariablesService.js'; import type { IdempotencyConfigOptions } from './types/IdempotencyOptions.js'; -import type { JMESPathParsingOptions } from '@aws-lambda-powertools/jmespath/types'; -import { PowertoolsFunctions } from '@aws-lambda-powertools/jmespath/functions'; /** * Configuration for the idempotency feature. @@ -52,9 +52,9 @@ class IdempotencyConfig { * @default false */ public throwOnNoIdempotencyKey: boolean; + /** * Use the local cache to store idempotency keys. - * @see {@link LRUCache} */ public useLocalCache: boolean; readonly #envVarsService: EnvironmentVariablesService; diff --git a/packages/idempotency/src/IdempotencyHandler.ts b/packages/idempotency/src/IdempotencyHandler.ts index ab2903063d..4dcfa03def 100644 --- a/packages/idempotency/src/IdempotencyHandler.ts +++ b/packages/idempotency/src/IdempotencyHandler.ts @@ -137,7 +137,7 @@ export class IdempotencyHandler { await this.#deleteInProgressRecord(); throw error; } - await this.#saveSuccessfullResult(result); + await this.#saveSuccessfulResult(result); return result; } @@ -208,7 +208,7 @@ export class IdempotencyHandler { * @param response The response returned by the handler. */ public async handleMiddyAfter(response: unknown): Promise { - await this.#saveSuccessfullResult(response as ReturnType); + await this.#saveSuccessfulResult(response as ReturnType); } /** @@ -401,7 +401,7 @@ export class IdempotencyHandler { * * @param result The result returned by the handler. */ - #saveSuccessfullResult = async (result: ReturnType): Promise => { + #saveSuccessfulResult = async (result: ReturnType): Promise => { try { await this.#persistenceStore.saveSuccess( this.#functionPayloadToBeHashed, diff --git a/packages/idempotency/src/idempotencyDecorator.ts b/packages/idempotency/src/idempotencyDecorator.ts index b997237486..64ad4c4fad 100644 --- a/packages/idempotency/src/idempotencyDecorator.ts +++ b/packages/idempotency/src/idempotencyDecorator.ts @@ -50,7 +50,9 @@ import type { * } * } * ``` - * @see {@link DynamoDBPersistenceLayer} + * + * @param options - Options to configure the idempotency behavior + * @see {@link persistence/DynamoDBPersistenceLayer.DynamoDBPersistenceLayer | DynamoDBPersistenceLayer} * @see https://www.typescriptlang.org/docs/handbook/decorators.html */ const idempotent = function ( diff --git a/packages/idempotency/src/types/IdempotencyOptions.ts b/packages/idempotency/src/types/IdempotencyOptions.ts index 3860c1f7eb..1b07b11218 100644 --- a/packages/idempotency/src/types/IdempotencyOptions.ts +++ b/packages/idempotency/src/types/IdempotencyOptions.ts @@ -7,7 +7,7 @@ import type { BasePersistenceLayer } from '../persistence/BasePersistenceLayer.j * Configuration options for the idempotency utility. * * When making a function idempotent you should always set - * a persistence store (i.e. {@link DynamoDBPersistenceLayer}). + * a persistence store (i.e. @see {@link persistence/DynamoDBPersistenceLayer.DynamoDBPersistenceLayer | DynamoDBPersistenceLayer}). * * Optionally, you can also pass a custom configuration object, * this allows you to customize the behavior of the idempotency utility. @@ -111,7 +111,7 @@ type ItempotentFunctionOptions> = T[1] extends Context * Options to configure the behavior of the idempotency logic. * * This is an internal type that is used by the Idempotency utility to - * configure {@link IdempotencyHandler}. + * configure. */ type IdempotencyHandlerOptions = { /** diff --git a/packages/idempotency/src/types/IdempotencyRecord.ts b/packages/idempotency/src/types/IdempotencyRecord.ts index 565ad8dac5..cfd01a44bf 100644 --- a/packages/idempotency/src/types/IdempotencyRecord.ts +++ b/packages/idempotency/src/types/IdempotencyRecord.ts @@ -1,9 +1,15 @@ import type { JSONValue } from '@aws-lambda-powertools/commons/types'; import type { IdempotencyRecordStatus } from '../constants.js'; +/** + * The status of an IdempotencyRecord + */ type IdempotencyRecordStatusValue = (typeof IdempotencyRecordStatus)[keyof typeof IdempotencyRecordStatus]; +/** + * Options for creating a new IdempotencyRecord + */ type IdempotencyRecordOptions = { idempotencyKey: string; status: IdempotencyRecordStatusValue; diff --git a/packages/idempotency/src/types/index.ts b/packages/idempotency/src/types/index.ts index 3f7d50ea34..96dfca108d 100644 --- a/packages/idempotency/src/types/index.ts +++ b/packages/idempotency/src/types/index.ts @@ -10,4 +10,5 @@ export type { IdempotencyConfigOptions, IdempotencyLambdaHandlerOptions, IdempotencyHandlerOptions, + ItempotentFunctionOptions, } from './IdempotencyOptions.js'; diff --git a/packages/idempotency/typedoc.json b/packages/idempotency/typedoc.json index e2ece84d17..d1bb1f2c03 100644 --- a/packages/idempotency/typedoc.json +++ b/packages/idempotency/typedoc.json @@ -7,7 +7,7 @@ "./src/types/index.ts", "./src/middleware/index.ts", "./src/persistence/index.ts", - "./src/persistence/DynamoDBPersistenceLayer.ts", + "./src/persistence/DynamoDBPersistenceLayer.ts" ], "readme": "README.md" } \ No newline at end of file From 707d1c1a185020a91b30533059b17a985036049e Mon Sep 17 00:00:00 2001 From: Alexander Schueren Date: Tue, 13 Aug 2024 13:28:15 +0200 Subject: [PATCH 03/10] add test example --- docs/utilities/idempotency.md | 284 ++++++++++++------ .../samples/testingIdempotency.json | 30 ++ .../idempotency/testingIdempotency.test.ts | 38 +++ .../idempotency/testingIdempotency.ts | 24 ++ 4 files changed, 292 insertions(+), 84 deletions(-) create mode 100644 examples/snippets/idempotency/samples/testingIdempotency.json create mode 100644 examples/snippets/idempotency/testingIdempotency.test.ts create mode 100644 examples/snippets/idempotency/testingIdempotency.ts diff --git a/docs/utilities/idempotency.md b/docs/utilities/idempotency.md index fb4bf2d42a..70bc7cab74 100644 --- a/docs/utilities/idempotency.md +++ b/docs/utilities/idempotency.md @@ -3,9 +3,8 @@ title: Idempotency description: Utility --- - - -The idempotency utility provides a simple solution to convert your Lambda functions into idempotent operations which are safe to retry. +The idempotency utility provides a simple solution to convert your Lambda functions into idempotent operations which are +safe to retry. ## Key features @@ -17,13 +16,17 @@ The idempotency utility provides a simple solution to convert your Lambda functi ## Terminology -The property of idempotency means that an operation does not cause additional side effects if it is called more than once with the same input parameters. +The property of idempotency means that an operation does not cause additional side effects if it is called more than +once with the same input parameters. -**Idempotent operations will return the same result when they are called multiple times with the same parameters**. This makes idempotent operations safe to retry. +**Idempotent operations will return the same result when they are called multiple times with the same parameters**. This +makes idempotent operations safe to retry. -**Idempotency key** is a hash representation of either the entire event or a specific configured subset of the event, and invocation results are **JSON serialized** and stored in your persistence storage layer. +**Idempotency key** is a hash representation of either the entire event or a specific configured subset of the event, +and invocation results are **JSON serialized** and stored in your persistence storage layer. -**Idempotency record** is the data representation of an idempotent request saved in your preferred storage layer. We use it to coordinate whether a request is idempotent, whether it's still valid or expired based on timestamps, etc. +**Idempotency record** is the data representation of an idempotent request saved in your preferred storage layer. We use +it to coordinate whether a request is idempotent, whether it's still valid or expired based on timestamps, etc.
```mermaid @@ -59,32 +62,41 @@ Install the library in your project npm i @aws-lambda-powertools/idempotency @aws-sdk/client-dynamodb @aws-sdk/lib-dynamodb ``` -While we support Amazon DynamoDB as a persistence layer out of the box, you need to bring your own AWS SDK for JavaScript v3 DynamoDB client. +While we support Amazon DynamoDB as a persistence layer out of the box, you need to bring your own AWS SDK for +JavaScript v3 DynamoDB client. ???+ note - This utility supports **[AWS SDK for JavaScript v3](https://docs.aws.amazon.com/AWSJavaScriptSDK/v3/latest/){target="_blank"} only**. If you are using the `nodejs18.x` runtime or newer, the AWS SDK for JavaScript v3 is already installed and you can install only the utility. +This utility supports **[AWS SDK for JavaScript v3](https://docs.aws.amazon.com/AWSJavaScriptSDK/v3/latest/){target=" +_blank"} only**. If you are using the `nodejs18.x` runtime or newer, the AWS SDK for JavaScript v3 is already installed +and you can install only the utility. ### IAM Permissions -Your Lambda function IAM Role must have `dynamodb:GetItem`, `dynamodb:PutItem`, `dynamodb:UpdateItem` and `dynamodb:DeleteItem` IAM permissions before using this feature. If you're using one of our examples: [AWS Serverless Application Model (SAM)](#required-resources) or [Terraform](#required-resources) the required permissions are already included. +Your Lambda function IAM Role must have `dynamodb:GetItem`, `dynamodb:PutItem`, `dynamodb:UpdateItem` +and `dynamodb:DeleteItem` IAM permissions before using this feature. If you're using one of our +examples: [AWS Serverless Application Model (SAM)](#required-resources) or [Terraform](#required-resources) the required +permissions are already included. ### Required resources -Before getting started, you need to create a persistent storage layer where the idempotency utility can store its state - your lambda functions will need read and write access to it. +Before getting started, you need to create a persistent storage layer where the idempotency utility can store its +state - your lambda functions will need read and write access to it. As of now, Amazon DynamoDB is the only supported persistent storage layer, so you'll need to create a table first. **Default table configuration** -If you're not [changing the default configuration for the DynamoDB persistence layer](#dynamodbpersistencelayer), this is the expected default configuration: +If you're not [changing the default configuration for the DynamoDB persistence layer](#dynamodbpersistencelayer), this +is the expected default configuration: | Configuration | Default value | Notes | -| ------------------ | :------------ | -------------------------------------------------------------------------------------- | +|--------------------|:--------------|----------------------------------------------------------------------------------------| | Partition key | `id` | The id of each idempotency record which a combination of `functionName#hashOfPayload`. | | TTL attribute name | `expiration` | This can only be configured after your table is created if you're using AWS Console. | ???+ tip "Tip: You can share a single state table for all functions" - You can reuse the same DynamoDB table to store idempotency state. We add the Lambda function name in addition to the idempotency key as a hash key. +You can reuse the same DynamoDB table to store idempotency state. We add the Lambda function name in addition to the +idempotency key as a hash key. === "AWS Cloud Development Kit (CDK) example" @@ -105,20 +117,28 @@ If you're not [changing the default configuration for the DynamoDB persistence l ``` ???+ warning "Warning: Large responses with DynamoDB persistence layer" - When using this utility with DynamoDB, your function's responses must be [smaller than 400KB](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Limits.html#limits-items){target="_blank"}. +When using this utility with DynamoDB, your function's responses must +be [smaller than 400KB](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Limits.html#limits-items) +{target="_blank"}. Larger items cannot be written to DynamoDB and will cause exceptions. ???+ info "Info: DynamoDB" - Each function invocation will make only 1 request to DynamoDB by using DynamoDB's [conditional expressions](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Expressions.ConditionExpressions.html){target="_blank"} to ensure that we don't overwrite existing records, - and [ReturnValuesOnConditionCheckFailure](https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_PutItem.html#DDB-PutItem-request-ReturnValuesOnConditionCheckFailure){target="_blank"} to return the record if it exists. - See [AWS Blog post on handling conditional write errors](https://aws.amazon.com/blogs/database/handle-conditional-write-errors-in-high-concurrency-scenarios-with-amazon-dynamodb/) for more details. - For retried invocations, you will see 1WCU and 1RCU. - Review the [DynamoDB pricing documentation](https://aws.amazon.com/dynamodb/pricing/){target="_blank"} to estimate the cost. +Each function invocation will make only 1 request to DynamoDB by using +DynamoDB's [conditional expressions](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Expressions.ConditionExpressions.html) +{target="_blank"} to ensure that we don't overwrite existing records, +and [ReturnValuesOnConditionCheckFailure](https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_PutItem.html#DDB-PutItem-request-ReturnValuesOnConditionCheckFailure) +{target="_blank"} to return the record if it exists. +See [AWS Blog post on handling conditional write errors](https://aws.amazon.com/blogs/database/handle-conditional-write-errors-in-high-concurrency-scenarios-with-amazon-dynamodb/) +for more details. +For retried invocations, you will see 1WCU and 1RCU. +Review the [DynamoDB pricing documentation](https://aws.amazon.com/dynamodb/pricing/){target="_blank"} to estimate the +cost. ### MakeIdempotent function wrapper -You can quickly start by initializing the `DynamoDBPersistenceLayer` class and using it with the `makeIdempotent` function wrapper on your Lambda handler. +You can quickly start by initializing the `DynamoDBPersistenceLayer` class and using it with the `makeIdempotent` +function wrapper on your Lambda handler. === "index.ts" @@ -132,19 +152,26 @@ You can quickly start by initializing the `DynamoDBPersistenceLayer` class and u --8<-- "examples/snippets/idempotency/types.ts:3:16" ``` -After processing this request successfully, a second request containing the exact same payload above will now return the same response, ensuring our customer isn't charged twice. +After processing this request successfully, a second request containing the exact same payload above will now return the +same response, ensuring our customer isn't charged twice. ???+ note - In this example, the entire Lambda handler is treated as a single idempotent operation. If your Lambda handler can cause multiple side effects, or you're only interested in making a specific logic idempotent, use the `makeIdempotent` high-order function only on the function that needs to be idempotent. +In this example, the entire Lambda handler is treated as a single idempotent operation. If your Lambda handler can cause +multiple side effects, or you're only interested in making a specific logic idempotent, use the `makeIdempotent` +high-order function only on the function that needs to be idempotent. See [Choosing a payload subset for idempotency](#choosing-a-payload-subset-for-idempotency) for more elaborate use cases. -You can also use the `makeIdempotent` function wrapper on any method that returns a response to make it idempotent. This is useful when you want to make a specific logic idempotent, for example when your Lambda handler performs multiple side effects and you only want to make a specific one idempotent. +You can also use the `makeIdempotent` function wrapper on any method that returns a response to make it idempotent. This +is useful when you want to make a specific logic idempotent, for example when your Lambda handler performs multiple side +effects and you only want to make a specific one idempotent. ???+ warning "Limitation" - Make sure to return a JSON serializable response from your function, otherwise you'll get an error. +Make sure to return a JSON serializable response from your function, otherwise you'll get an error. -When using `makeIdempotent` on arbitrary functions, you can tell us which argument in your function signature has the data we should use via **`dataIndexArgument`**. If you don't specify this argument, we'll use the first argument in the function signature. +When using `makeIdempotent` on arbitrary functions, you can tell us which argument in your function signature has the +data we should use via **`dataIndexArgument`**. If you don't specify this argument, we'll use the first argument in the +function signature. === "index.ts" @@ -158,14 +185,20 @@ When using `makeIdempotent` on arbitrary functions, you can tell us which argume --8<-- "examples/snippets/idempotency/types.ts:3:16" ``` -The function this example has two arguments, note that while wrapping it with the `makeIdempotent` high-order function, we specify the `dataIndexArgument` as `1` to tell the decorator that the second argument is the one with the data we should use to make the function idempotent. Remember that arguments are zero-indexed, so the first argument is `0`, the second is `1`, etc. +The function this example has two arguments, note that while wrapping it with the `makeIdempotent` high-order function, +we specify the `dataIndexArgument` as `1` to tell the decorator that the second argument is the one with the data we +should use to make the function idempotent. Remember that arguments are zero-indexed, so the first argument is `0`, the +second is `1`, etc. ### Idempotent Decorator -You can also use the `@idempotent` decorator to make your Lambda handler idempotent, similar to the `makeIdempotent` function wrapper. +You can also use the `@idempotent` decorator to make your Lambda handler idempotent, similar to the `makeIdempotent` +function wrapper. !!! info - The class method decorators in this project follow the experimental implementation enabled via the [`experimentalDecorators` compiler option](https://www.typescriptlang.org/tsconfig#experimentalDecorators) in TypeScript. +The class method decorators in this project follow the experimental implementation enabled via +the [`experimentalDecorators` compiler option](https://www.typescriptlang.org/tsconfig#experimentalDecorators) in +TypeScript. Additionally, they are implemented to decorate async methods. When decorating a synchronous one, the decorator replaces its implementation with an async one causing the caller to have to `await` the now decorated method. @@ -183,18 +216,26 @@ You can also use the `@idempotent` decorator to make your Lambda handler idempot --8<-- "examples/snippets/idempotency/types.ts" ``` -You can use the decorator on your Lambda handler or on any function that returns a response to make it idempotent. This is useful when you want to make a specific logic idempotent, for example when your Lambda handler performs multiple side effects and you only want to make a specific one idempotent. -The configuration options for the `@idempotent` decorator are the same as the ones for the `makeIdempotent` function wrapper. +You can use the decorator on your Lambda handler or on any function that returns a response to make it idempotent. This +is useful when you want to make a specific logic idempotent, for example when your Lambda handler performs multiple side +effects and you only want to make a specific one idempotent. +The configuration options for the `@idempotent` decorator are the same as the ones for the `makeIdempotent` function +wrapper. ### MakeHandlerIdempotent Middy middleware !!! tip "A note about Middy" - We guarantee support for both Middy.js `v4.x` & `v5.x` with the latter being available only if you are using ES modules. - Check their docs to learn more about [Middy and its middleware stack](https://middy.js.org/docs/intro/getting-started){target="_blank"} as well as [best practices when working with Powertools](https://middy.js.org/docs/integrations/lambda-powertools#best-practices){target="_blank"}. +We guarantee support for both Middy.js `v4.x` & `v5.x` with the latter being available only if you are using ES modules. +Check their docs to learn more about [Middy and its middleware stack](https://middy.js.org/docs/intro/getting-started) +{target="_blank"} as well +as [best practices when working with Powertools](https://middy.js.org/docs/integrations/lambda-powertools#best-practices) +{target="_blank"}. -If you are using [Middy.js](https://middy.js.org){target="_blank"} as your middleware engine, you can use the `makeHandlerIdempotent` middleware to make your Lambda handler idempotent. +If you are using [Middy.js](https://middy.js.org){target="_blank"} as your middleware engine, you can use +the `makeHandlerIdempotent` middleware to make your Lambda handler idempotent. -Similar to the `makeIdempotent` function wrapper, you can quickly make your Lambda handler idempotent by initializing the `DynamoDBPersistenceLayer` class and using it with the `makeHandlerIdempotent` middleware. +Similar to the `makeIdempotent` function wrapper, you can quickly make your Lambda handler idempotent by initializing +the `DynamoDBPersistenceLayer` class and using it with the `makeHandlerIdempotent` middleware. === "index.ts" @@ -210,18 +251,25 @@ Similar to the `makeIdempotent` function wrapper, you can quickly make your Lamb ### Choosing a payload subset for idempotency -Use [`IdempotencyConfig`](#customizing-the-default-behavior) to instruct the idempotent decorator to only use a portion of your payload to verify whether a request is idempotent, and therefore it should not be retried. When dealing with a more elaborate payload, where parts of the payload always change, you should use the **`eventKeyJmesPath`** parameter. +Use [`IdempotencyConfig`](#customizing-the-default-behavior) to instruct the idempotent decorator to only use a portion +of your payload to verify whether a request is idempotent, and therefore it should not be retried. When dealing with a +more elaborate payload, where parts of the payload always change, you should use the **`eventKeyJmesPath`** parameter. **Payment scenario** -In this example, we have a Lambda handler that creates a payment for a user subscribing to a product. We want to ensure that we don't accidentally charge our customer by subscribing them more than once. +In this example, we have a Lambda handler that creates a payment for a user subscribing to a product. We want to ensure +that we don't accidentally charge our customer by subscribing them more than once. -Imagine the function executes successfully, but the client never receives the response due to a connection issue. It is safe to retry in this instance, as the idempotent decorator will return a previously saved response. +Imagine the function executes successfully, but the client never receives the response due to a connection issue. It is +safe to retry in this instance, as the idempotent decorator will return a previously saved response. -**What we want here** is to instruct Idempotency to use the `user` and `productId` fields from our incoming payload as our idempotency key. If we were to treat the entire request as our idempotency key, a simple HTTP header or timestamp change would cause our customer to be charged twice. +**What we want here** is to instruct Idempotency to use the `user` and `productId` fields from our incoming payload as +our idempotency key. If we were to treat the entire request as our idempotency key, a simple HTTP header or timestamp +change would cause our customer to be charged twice. ???+ warning "Deserializing JSON strings in payloads for increased accuracy." - The payload extracted by the `eventKeyJmesPath` is treated as a string by default. This means there could be differences in whitespace even when the JSON payload itself is identical. +The payload extracted by the `eventKeyJmesPath` is treated as a string by default. This means there could be differences +in whitespace even when the JSON payload itself is identical. To alter this behaviour, we can use the [JMESPath built-in function `powertools_json()`](jmespath.md#powertools_json-function) to treat the payload as a JSON object rather than a string. @@ -245,16 +293,23 @@ Imagine the function executes successfully, but the client never receives the re ### Lambda timeouts -To prevent against extended failed retries when a [Lambda function times out](https://aws.amazon.com/premiumsupport/knowledge-center/lambda-verify-invocation-timeouts/), Powertools for AWS Lambda calculates and includes the remaining invocation available time as part of the idempotency record. -This is automatically done when you wrap your Lambda handler with the [makeIdempotent](#makeidempotent-function-wrapper) function wrapper, or use the [`makeHandlerIdempotent`](#makehandleridempotent-middy-middleware) Middy middleware. +To prevent against extended failed retries when +a [Lambda function times out](https://aws.amazon.com/premiumsupport/knowledge-center/lambda-verify-invocation-timeouts/), +Powertools for AWS Lambda calculates and includes the remaining invocation available time as part of the idempotency +record. +This is automatically done when you wrap your Lambda handler with the [makeIdempotent](#makeidempotent-function-wrapper) +function wrapper, or use the [`makeHandlerIdempotent`](#makehandleridempotent-middy-middleware) Middy middleware. ???+ example - If a second invocation happens **after** this timestamp, and the record is marked as `INPROGRESS`, we will execute the invocation again as if it was in the `EXPIRED` state (e.g, `expire_seconds` field elapsed). +If a second invocation happens **after** this timestamp, and the record is marked as `INPROGRESS`, we will execute the +invocation again as if it was in the `EXPIRED` state (e.g, `expire_seconds` field elapsed). This means that if an invocation expired during execution, it will be quickly executed again on the next retry. ???+ important - If you are only using the [makeIdempotent function wrapper](#makeidempotent-function-wrapper) to guard isolated parts of your code outside of your handler, you must use `registerLambdaContext` available in the [idempotency config object](#customizing-the-default-behavior) to benefit from this protection. +If you are only using the [makeIdempotent function wrapper](#makeidempotent-function-wrapper) to guard isolated parts of +your code outside of your handler, you must use `registerLambdaContext` available in +the [idempotency config object](#customizing-the-default-behavior) to benefit from this protection. Here is an example on how you register the Lambda context in your handler: @@ -266,8 +321,10 @@ Here is an example on how you register the Lambda context in your handler: ### Handling exceptions -If you are making on your entire Lambda handler idempotent, any unhandled exceptions that are raised during the code execution will cause **the record in the persistence layer to be deleted**. -This means that new invocations will execute your code again despite having the same payload. If you don't want the record to be deleted, you need to catch exceptions within the idempotent function and return a successful response. +If you are making on your entire Lambda handler idempotent, any unhandled exceptions that are raised during the code +execution will cause **the record in the persistence layer to be deleted**. +This means that new invocations will execute your code again despite having the same payload. If you don't want the +record to be deleted, you need to catch exceptions within the idempotent function and return a successful response.
```mermaid @@ -288,9 +345,13 @@ sequenceDiagram Idempotent sequence exception
-If you are using `makeIdempotent` on any other function, any unhandled exceptions that are thrown _inside_ the wrapped function will cause the record in the persistence layer to be deleted, and allow the function to be executed again if retried. +If you are using `makeIdempotent` on any other function, any unhandled exceptions that are thrown _inside_ the wrapped +function will cause the record in the persistence layer to be deleted, and allow the function to be executed again if +retried. -If an error is thrown _outside_ the scope of the decorated function and after your function has been called, the persistent record will not be affected. In this case, idempotency will be maintained for your decorated function. Example: +If an error is thrown _outside_ the scope of the decorated function and after your function has been called, the +persistent record will not be affected. In this case, idempotency will be maintained for your decorated function. +Example: === "Handling exceptions" @@ -299,7 +360,7 @@ If an error is thrown _outside_ the scope of the decorated function and after yo ``` ???+ warning - **We will throw `IdempotencyPersistenceLayerError`** if any of the calls to the persistence layer fail unexpectedly. +**We will throw `IdempotencyPersistenceLayerError`** if any of the calls to the persistence layer fail unexpectedly. As this happens outside the scope of your decorated function, you are not able to catch it when making your Lambda handler idempotent. @@ -509,7 +570,8 @@ sequenceDiagram #### DynamoDBPersistenceLayer -This persistence layer is built-in, and you can either use an existing DynamoDB table or create a new one dedicated for idempotency state (recommended). +This persistence layer is built-in, and you can either use an existing DynamoDB table or create a new one dedicated for +idempotency state (recommended). === "Customizing DynamoDBPersistenceLayer to suit your table structure" @@ -517,10 +579,11 @@ This persistence layer is built-in, and you can either use an existing DynamoDB --8<-- "examples/snippets/idempotency/customizePersistenceLayer.ts" ``` -When using DynamoDB as a persistence layer, you can alter the attribute names by passing these parameters when initializing the persistence layer: +When using DynamoDB as a persistence layer, you can alter the attribute names by passing these parameters when +initializing the persistence layer: | Parameter | Required | Default | Description | -| ------------------------ | ------------------ | ------------------------------------ | -------------------------------------------------------------------------------------------------------- | +|--------------------------|--------------------|--------------------------------------|----------------------------------------------------------------------------------------------------------| | **tableName** | :heavy_check_mark: | | Table name to store state | | **keyAttr** | | `id` | Partition key of the table. Hashed representation of the payload (unless **sort_key_attr** is specified) | | **expiryAttr** | | `expiration` | Unix timestamp of when record expires | @@ -533,10 +596,11 @@ When using DynamoDB as a persistence layer, you can alter the attribute names by ### Customizing the default behavior -Idempotent decorator can be further configured with **`IdempotencyConfig`** as seen in the previous examples. These are the available options for further configuration +Idempotent decorator can be further configured with **`IdempotencyConfig`** as seen in the previous examples. These are +the available options for further configuration | Parameter | Default | Description | -| ----------------------------- | ------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +|-------------------------------|---------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | **eventKeyJmespath** | `''` | JMESPath expression to extract the idempotency key from the event | | **payloadValidationJmespath** | `''` | JMESPath expression to validate whether certain parameters have changed in the event while the event payload | | **throwOnNoIdempotencyKey** | `false` | Throw an error if no idempotency key was found in the request | @@ -547,19 +611,23 @@ Idempotent decorator can be further configured with **`IdempotencyConfig`** as s ### Handling concurrent executions with the same payload -This utility will throw an **`IdempotencyAlreadyInProgressError`** error if you receive **multiple invocations with the same payload while the first invocation hasn't completed yet**. +This utility will throw an **`IdempotencyAlreadyInProgressError`** error if you receive **multiple invocations with the +same payload while the first invocation hasn't completed yet**. ???+ info - If you receive `IdempotencyAlreadyInProgressError`, you can safely retry the operation. +If you receive `IdempotencyAlreadyInProgressError`, you can safely retry the operation. -This is a locking mechanism for correctness. Since we don't know the result from the first invocation yet, we can't safely allow another concurrent execution. +This is a locking mechanism for correctness. Since we don't know the result from the first invocation yet, we can't +safely allow another concurrent execution. ### Using in-memory cache -**By default, in-memory local caching is disabled**, since we don't know how much memory you consume per invocation compared to the maximum configured in your Lambda function. +**By default, in-memory local caching is disabled**, since we don't know how much memory you consume per invocation +compared to the maximum configured in your Lambda function. ???+ note "Note: This in-memory cache is local to each Lambda execution environment" - This means it will be effective in cases where your function's concurrency is low in comparison to the number of "retry" invocations with the same payload, because cache might be empty. +This means it will be effective in cases where your function's concurrency is low in comparison to the number of "retry" +invocations with the same payload, because cache might be empty. You can enable in-memory caching with the **`useLocalCache`** parameter: @@ -569,13 +637,15 @@ You can enable in-memory caching with the **`useLocalCache`** parameter: --8<-- "examples/snippets/idempotency/workingWithLocalCache.ts" ``` -When enabled, the default is to cache a maximum of 256 records in each Lambda execution environment - You can change it with the **`maxLocalCacheSize`** parameter. +When enabled, the default is to cache a maximum of 256 records in each Lambda execution environment - You can change it +with the **`maxLocalCacheSize`** parameter. ### Expiring idempotency records !!! note "By default, we expire idempotency records after **an hour** (3600 seconds)." -In most cases, it is not desirable to store the idempotency records forever. Rather, you want to guarantee that the same payload won't be executed within a period of time. +In most cases, it is not desirable to store the idempotency records forever. Rather, you want to guarantee that the same +payload won't be executed within a period of time. You can change this window with the **`expiresAfterSeconds`** parameter: @@ -585,10 +655,12 @@ You can change this window with the **`expiresAfterSeconds`** parameter: --8<-- "examples/snippets/idempotency/workingWithRecordExpiration.ts" ``` -This will mark any records older than 5 minutes as expired, and [your function will be executed as normal if it is invoked with a matching payload](#expired-idempotency-records). +This will mark any records older than 5 minutes as expired, +and [your function will be executed as normal if it is invoked with a matching payload](#expired-idempotency-records). ???+ important "Idempotency record expiration vs DynamoDB time-to-live (TTL)" - [DynamoDB TTL is a feature](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/howitworks-ttl.html){target="_blank"} to remove items after a certain period of time, it may occur within 48 hours of expiration. +[DynamoDB TTL is a feature](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/howitworks-ttl.html) +{target="_blank"} to remove items after a certain period of time, it may occur within 48 hours of expiration. We don't rely on DynamoDB or any persistence storage layer to determine whether a record is expired to avoid eventual inconsistency states. @@ -600,12 +672,16 @@ This will mark any records older than 5 minutes as expired, and [your function w ### Payload validation -???+ question "Question: What if your function is invoked with the same payload except some outer parameters have changed?" - Example: A payment transaction for a given productID was requested twice for the same customer, **however the amount to be paid has changed in the second transaction**. +???+ question "Question: What if your function is invoked with the same payload except some outer parameters have +changed?" +Example: A payment transaction for a given productID was requested twice for the same customer, **however the amount to +be paid has changed in the second transaction**. -By default, we will return the same result as it returned before, however in this instance it may be misleading; we provide a fail fast payload validation to address this edge case. +By default, we will return the same result as it returned before, however in this instance it may be misleading; we +provide a fail fast payload validation to address this edge case. -With **`payloadValidationJmesPath`**, you can provide an additional JMESPath expression to specify which part of the event body should be validated against previous idempotent invocations +With **`payloadValidationJmesPath`**, you can provide an additional JMESPath expression to specify which part of the +event body should be validated against previous idempotent invocations === "Payload validation" @@ -613,23 +689,29 @@ With **`payloadValidationJmesPath`**, you can provide an additional JMESPath exp --8<-- "examples/snippets/idempotency/workingWithPayloadValidation.ts" ``` -In this example, the **`userId`** and **`productId`** keys are used as the payload to generate the idempotency key, as per **`eventKeyJmespath`** parameter. +In this example, the **`userId`** and **`productId`** keys are used as the payload to generate the idempotency key, as +per **`eventKeyJmespath`** parameter. ???+ note - If we try to send the same request but with a different amount, we will raise **`IdempotencyValidationError`**. +If we try to send the same request but with a different amount, we will raise **`IdempotencyValidationError`**. -Without payload validation, we would have returned the same result as we did for the initial request. Since we're also returning an amount in the response, this could be quite confusing for the client. +Without payload validation, we would have returned the same result as we did for the initial request. Since we're also +returning an amount in the response, this could be quite confusing for the client. -By using **`payloadValidationJmesPath="amount"`**, we prevent this potentially confusing behavior and instead throw an error. +By using **`payloadValidationJmesPath="amount"`**, we prevent this potentially confusing behavior and instead throw an +error. ### Making idempotency key required If you want to enforce that an idempotency key is required, you can set **`throwOnNoIdempotencyKey`** to `true`. -This means that we will raise **`IdempotencyKeyError`** if the evaluation of **`eventKeyJmesPath`** results in an empty subset. +This means that we will raise **`IdempotencyKeyError`** if the evaluation of **`eventKeyJmesPath`** results in an empty +subset. ???+ warning - To prevent errors, transactions will not be treated as idempotent if **`throwOnNoIdempotencyKey`** is set to `false` and the evaluation of **`eventKeyJmesPath`** is an empty result. Therefore, no data will be fetched, stored, or deleted in the idempotency storage layer. +To prevent errors, transactions will not be treated as idempotent if **`throwOnNoIdempotencyKey`** is set to `false` and +the evaluation of **`eventKeyJmesPath`** is an empty result. Therefore, no data will be fetched, stored, or deleted in +the idempotency storage layer. === "Idempotency key required" @@ -652,11 +734,14 @@ This means that we will raise **`IdempotencyKeyError`** if the evaluation of **` ### Batch integration You can easily integrate with [Batch](batch.md) utility by using idempotency wrapper around your processing function. -This ensures that you process each record in an idempotent manner, and guard against a [Lambda timeout](#lambda-timeouts) idempotent situation. +This ensures that you process each record in an idempotent manner, and guard against +a [Lambda timeout](#lambda-timeouts) idempotent situation. ???+ "Choosing a unique batch record attribute" - In this example, we choose `messageId` as our idempotency key since we know it'll be unique. - Depending on your use case, it might be more accurate [to choose another field](#choosing-a-payload-subset-for-idempotency) your producer intentionally set to define uniqueness. +In this example, we choose `messageId` as our idempotency key since we know it'll be unique. +Depending on your use case, it might be more +accurate [to choose another field](#choosing-a-payload-subset-for-idempotency) your producer intentionally set to define +uniqueness. === "Integration with batch processor" @@ -672,7 +757,9 @@ This ensures that you process each record in an idempotent manner, and guard aga ### Customizing AWS SDK configuration -The **`clientConfig`** and **`awsSdkV3Client`** parameters enable you to pass in custom configurations or your own [DynamoDBClient](https://docs.aws.amazon.com/AWSJavaScriptSDK/v3/latest/client/dynamodb/){target="_blank"} when constructing the persistence store. +The **`clientConfig`** and **`awsSdkV3Client`** parameters enable you to pass in custom configurations or your +own [DynamoDBClient](https://docs.aws.amazon.com/AWSJavaScriptSDK/v3/latest/client/dynamodb/){target="_blank"} when +constructing the persistence store. === "Passing specific configuration" @@ -688,9 +775,11 @@ The **`clientConfig`** and **`awsSdkV3Client`** parameters enable you to pass in ### Using a DynamoDB table with a composite primary key -When using a composite primary key table (hash+range key), use `sortKeyAttr` parameter when initializing your persistence layer. +When using a composite primary key table (hash+range key), use `sortKeyAttr` parameter when initializing your +persistence layer. -With this setting, we will save the idempotency key in the sort key instead of the primary key. By default, the primary key will now be set to `idempotency#{LAMBDA_FUNCTION_NAME}`. +With this setting, we will save the idempotency key in the sort key instead of the primary key. By default, the primary +key will now be set to `idempotency#{LAMBDA_FUNCTION_NAME}`. You can optionally set a static value for the partition key using the `staticPkValue` parameter. @@ -703,7 +792,7 @@ You can optionally set a static value for the partition key using the `staticPkV The example function above would cause data to be stored in DynamoDB like this: | id | sort_key | expiration | status | data | -| ---------------------------- | -------------------------------- | ---------- | ----------- | ---------------------------------------------------------------- | +|------------------------------|----------------------------------|------------|-------------|------------------------------------------------------------------| | idempotency#MyLambdaFunction | 1e956ef7da78d0cb890be999aecc0c9e | 1636549553 | COMPLETED | {"paymentId": "12345, "message": "success", "statusCode": 200} | | idempotency#MyLambdaFunction | 2b2cdb5f86361e97b4383087c1ffdf27 | 1636549571 | COMPLETED | {"paymentId": "527212", "message": "success", "statusCode": 200} | | idempotency#MyLambdaFunction | f091d2527ad1c78f05d54cc3f363be80 | 1636549585 | IN_PROGRESS | | @@ -712,10 +801,13 @@ The example function above would cause data to be stored in DynamoDB like this: This utility provides an abstract base class (ABC), so that you can implement your choice of persistent storage layer. -You can create your own persistent store from scratch by inheriting the `BasePersistenceLayer` class, and implementing `_getRecord()`, `_putRecord()`, `_updateRecord()` and `_deleteRecord()`. +You can create your own persistent store from scratch by inheriting the `BasePersistenceLayer` class, and +implementing `_getRecord()`, `_putRecord()`, `_updateRecord()` and `_deleteRecord()`. -* `_getRecord()` – Retrieves an item from the persistence store using an idempotency key and returns it as a `IdempotencyRecord` instance. -* `_putRecord()` – Adds a `IdempotencyRecord` to the persistence store if it doesn't already exist with that key. Throws an `IdempotencyItemAlreadyExistsError` error if a non-expired entry already exists. +* `_getRecord()` – Retrieves an item from the persistence store using an idempotency key and returns it as + a `IdempotencyRecord` instance. +* `_putRecord()` – Adds a `IdempotencyRecord` to the persistence store if it doesn't already exist with that key. Throws + an `IdempotencyItemAlreadyExistsError` error if a non-expired entry already exists. * `_updateRecord()` – Updates an item in the persistence store. * `_deleteRecord()` – Removes an item from the persistence store. @@ -740,10 +832,34 @@ Below an example implementation of a custom persistence layer backed by a generi ``` ???+ danger - Pay attention to the documentation for each - you may need to perform additional checks inside these methods to ensure the idempotency guarantees remain intact. +Pay attention to the documentation for each - you may need to perform additional checks inside these methods to ensure +the idempotency guarantees remain intact. For example, the `_putRecord()` method needs to throw an error if a non-expired record already exists in the data store with a matching key. +## Testing your code + +When testing idempotency, you can verify that your function stores the correct data in the persistence layer. +In this example, we send two different requests with the same payload. + +=== "index.ts" + + ```typescript + --8<-- "examples/snippets/idempotency/testingIdempotency.ts" + ``` + +=== "index.test.ts" + + ```typescript + --8<-- "examples/snippets/idempotency/testingIdempotency.test.ts" + ``` + +=== "Sample event" + + ```json + --8<-- "examples/snippets/idempotency/samples/testingIdempotency.json" + ``` + ## Extra resources If you're interested in a deep dive on how Amazon uses idempotency when building our APIs, check out diff --git a/examples/snippets/idempotency/samples/testingIdempotency.json b/examples/snippets/idempotency/samples/testingIdempotency.json new file mode 100644 index 0000000000..158b3a874f --- /dev/null +++ b/examples/snippets/idempotency/samples/testingIdempotency.json @@ -0,0 +1,30 @@ +{ + "version": "2.0", + "routeKey": "ANY /createpayment", + "rawPath": "/createpayment", + "rawQueryString": "", + "headers": { + "Header1": "value1", + "X-Idempotency-Key": "abcdefg" + }, + "requestContext": { + "accountId": "123456789012", + "apiId": "api-id", + "domainName": "id.execute-api.us-east-1.amazonaws.com", + "domainPrefix": "id", + "http": { + "method": "POST", + "path": "/createpayment", + "protocol": "HTTP/1.1", + "sourceIp": "ip", + "userAgent": "agent" + }, + "requestId": "id", + "routeKey": "ANY /createpayment", + "stage": "$default", + "time": "10/Feb/2021:13:40:43 +0000", + "timeEpoch": 1612964443723 + }, + "body": "{\"user\":\"xyz\",\"productId\":\"123456789\"}", + "isBase64Encoded": false +} diff --git a/examples/snippets/idempotency/testingIdempotency.test.ts b/examples/snippets/idempotency/testingIdempotency.test.ts new file mode 100644 index 0000000000..499e08b004 --- /dev/null +++ b/examples/snippets/idempotency/testingIdempotency.test.ts @@ -0,0 +1,38 @@ +import type { Context } from 'aws-lambda'; +import { DocumentClient } from 'aws-sdk/clients/dynamodb'; +import * as API_GATEWAY_EXAMPLE_EVENT from './samples/testingIdempotency.json'; +import { idempotentHandler } from './testingIdempotency'; + +describe('Idempotent Handler', () => { + const ddb = new DocumentClient({}); + + it('should return the same response for the same request', async () => { + // given + const context = {} as Context; + + const firstRequest = API_GATEWAY_EXAMPLE_EVENT; + + // modify time field to simulate a different request + const secondRequest = { + ...API_GATEWAY_EXAMPLE_EVENT, + requestContext: { + ...API_GATEWAY_EXAMPLE_EVENT.requestContext, + time: 1612964493723, + }, + }; + + // when + const firstResponse = await idempotentHandler(firstRequest, context); + const secondResponse = await idempotentHandler(secondRequest, context); + // then + expect(firstResponse).toEqual(secondResponse); + // check if we only have one item in the table + const idempotencyRecords = await ddb + .scan({ TableName: 'idempotency-store' }) + .promise(); + + expect(idempotencyRecords.Items).toHaveLength(1); + expect(idempotencyRecords.Items?.[0].status).toEqual('COMPLETED'); + expect(idempotencyRecords.Items?.[0].data).toEqual(firstResponse); + }); +}); diff --git a/examples/snippets/idempotency/testingIdempotency.ts b/examples/snippets/idempotency/testingIdempotency.ts new file mode 100644 index 0000000000..02c7a54c48 --- /dev/null +++ b/examples/snippets/idempotency/testingIdempotency.ts @@ -0,0 +1,24 @@ +import { + IdempotencyConfig, + makeIdempotent, +} from '@aws-lambda-powertools/idempotency'; +import { DynamoDBPersistenceLayer } from '@aws-lambda-powertools/idempotency/dynamodb'; + +const idempotencyConfig = new IdempotencyConfig({}); +const persistenceStore = new DynamoDBPersistenceLayer({ + tableName: 'idempotency-store', +}); + +const handler = async (event: unknown, context: unknown) => { + return { + statusCode: 200, + body: JSON.stringify({ message: 'Success', event: event }), + }; +}; + +const idempotentHandler = makeIdempotent(handler, { + config: idempotencyConfig, + persistenceStore: persistenceStore, +}); + +export { idempotentHandler }; From e89e1652221274b6be9b1e0d008bee16c6c30115 Mon Sep 17 00:00:00 2001 From: Alexander Schueren Date: Tue, 13 Aug 2024 13:30:16 +0200 Subject: [PATCH 04/10] add link to e2e tests --- docs/utilities/idempotency.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/utilities/idempotency.md b/docs/utilities/idempotency.md index 70bc7cab74..a9cf3a22fd 100644 --- a/docs/utilities/idempotency.md +++ b/docs/utilities/idempotency.md @@ -860,6 +860,9 @@ In this example, we send two different requests with the same payload. --8<-- "examples/snippets/idempotency/samples/testingIdempotency.json" ``` +You can also check how +we [tested idempotency in our end to end tests](https://github.com/aws-powertools/powertools-lambda-typescript/tree/main/packages/idempotency/tests/e2e). + ## Extra resources If you're interested in a deep dive on how Amazon uses idempotency when building our APIs, check out From 130d7d86343e0822acf7b08915bbb0721f83e3ea Mon Sep 17 00:00:00 2001 From: Alexander Schueren Date: Tue, 13 Aug 2024 13:36:02 +0200 Subject: [PATCH 05/10] remove ignores and toc --- .markdownlintignore | 1 - packages/idempotency/README.md | 294 +++++++++++++++++---------------- 2 files changed, 156 insertions(+), 139 deletions(-) diff --git a/.markdownlintignore b/.markdownlintignore index 73eedf3c15..2cac804a00 100644 --- a/.markdownlintignore +++ b/.markdownlintignore @@ -13,7 +13,6 @@ LICENSE # these will be removed from the ignore and linted in future PRs packages/batch/README.md packages/commons/README.md -packages/idempotency/README.md packages/jmespath/README.md packages/logger/README.md packages/metrics/README.md diff --git a/packages/idempotency/README.md b/packages/idempotency/README.md index 3b26db7e4c..3212cdd1f7 100644 --- a/packages/idempotency/README.md +++ b/packages/idempotency/README.md @@ -1,30 +1,18 @@ -# Powertools for AWS Lambda (TypeScript) - Idempotency Utility +# Powertools for AWS Lambda (TypeScript) - Idempotency Utility -Powertools for AWS Lambda (TypeScript) is a developer toolkit to implement Serverless [best practices and increase developer velocity](https://docs.powertools.aws.dev/lambda/typescript/latest/#features). +Powertools for AWS Lambda (TypeScript) is a developer toolkit to implement +Serverless [best practices and increase developer velocity](https://docs.powertools.aws.dev/lambda/typescript/latest/#features). You can use the package in both TypeScript and JavaScript code bases. -- [Intro](#intro) -- [Usage](#usage) - - [Function wrapper](#function-wrapper) - - [Decorator](#decorator) - - [Middy middleware](#middy-middleware) - - [DynamoDB persistence layer](#dynamodb-persistence-layer) -- [Contribute](#contribute) -- [Roadmap](#roadmap) -- [Connect](#connect) -- [How to support Powertools for AWS Lambda (TypeScript)?](#how-to-support-powertools-for-aws-lambda-typescript) - - [Becoming a reference customer](#becoming-a-reference-customer) - - [Sharing your work](#sharing-your-work) - - [Using Lambda Layer](#using-lambda-layer) -- [License](#license) - ## Intro This package provides a utility to implement idempotency in your Lambda functions. -You can either use it to wrap a function, decorate a method, or as Middy middleware to make your AWS Lambda handler idempotent. +You can either use it to wrap a function, decorate a method, or as Middy middleware to make your AWS Lambda handler +idempotent. -The current implementation provides a persistence layer for Amazon DynamoDB, which offers a variety of configuration options. You can also bring your own persistence layer by extending the `BasePersistenceLayer` class. +The current implementation provides a persistence layer for Amazon DynamoDB, which offers a variety of configuration +options. You can also bring your own persistence layer by extending the `BasePersistenceLayer` class. ## Usage @@ -34,146 +22,156 @@ To get started, install the library by running: npm i @aws-lambda-powertools/idempotency @aws-sdk/client-dynamodb @aws-sdk/lib-dynamodb ``` -Next, review the IAM permissions attached to your AWS Lambda function and make sure you allow the [actions detailed](https://docs.powertools.aws.dev/lambda/typescript/latest/utilities/idempotency/#iam-permissions) in the documentation of the utility. +Next, review the IAM permissions attached to your AWS Lambda function and make sure you allow +the [actions detailed](https://docs.powertools.aws.dev/lambda/typescript/latest/utilities/idempotency/#iam-permissions) +in the documentation of the utility. ### Function wrapper -You can make any function idempotent, and safe to retry, by wrapping it using the `makeIdempotent` higher-order function. +You can make any function idempotent, and safe to retry, by wrapping it using the `makeIdempotent` higher-order +function. -The `makeIdempotent` function takes a reference to the function to be made idempotent as first argument, and an object with options as second argument. +The `makeIdempotent` function takes a reference to the function to be made idempotent as first argument, and an object +with options as second argument. -When you wrap your Lambda handler function, the utility uses the content of the `event` parameter to handle the idempotency logic. +When you wrap your Lambda handler function, the utility uses the content of the `event` parameter to handle the +idempotency logic. ```ts -import { makeIdempotent } from '@aws-lambda-powertools/idempotency'; -import { DynamoDBPersistenceLayer } from '@aws-lambda-powertools/idempotency/dynamodb'; -import type { Context, APIGatewayProxyEvent } from 'aws-lambda'; +import {makeIdempotent} from '@aws-lambda-powertools/idempotency'; +import {DynamoDBPersistenceLayer} from '@aws-lambda-powertools/idempotency/dynamodb'; +import type {Context, APIGatewayProxyEvent} from 'aws-lambda'; + `` const persistenceStore = new DynamoDBPersistenceLayer({ - tableName: 'idempotencyTableName', + tableName: 'idempotencyTableName', }); const myHandler = async ( - event: APIGatewayProxyEvent, - _context: Context + event: APIGatewayProxyEvent, + _context: Context ): Promise => { - // your code goes here here + // your code goes here here }; export const handler = makeIdempotent(myHandler, { - persistenceStore, + persistenceStore, }); ``` You can also use the `makeIdempotent` function to wrap any other arbitrary function, not just Lambda handlers. ```ts -import { makeIdempotent } from '@aws-lambda-powertools/idempotency'; -import { DynamoDBPersistenceLayer } from '@aws-lambda-powertools/idempotency/dynamodb'; -import type { Context, SQSEvent, SQSRecord } from 'aws-lambda'; +import {makeIdempotent} from '@aws-lambda-powertools/idempotency'; +import {DynamoDBPersistenceLayer} from '@aws-lambda-powertools/idempotency/dynamodb'; +import type {Context, SQSEvent, SQSRecord} from 'aws-lambda'; const persistenceStore = new DynamoDBPersistenceLayer({ - tableName: 'idempotencyTableName', + tableName: 'idempotencyTableName', }); const processingFunction = async (payload: SQSRecord): Promise => { - // your code goes here here + // your code goes here here }; const processIdempotently = makeIdempotent(processingFunction, { - persistenceStore, + persistenceStore, }); export const handler = async ( - event: SQSEvent, - _context: Context + event: SQSEvent, + _context: Context ): Promise => { - for (const record of event.Records) { - await processIdempotently(record); - } + for (const record of event.Records) { + await processIdempotently(record); + } }; ``` -If your function has multiple arguments, you can use the `dataIndexArgument` option to specify which argument should be used as the idempotency key. +If your function has multiple arguments, you can use the `dataIndexArgument` option to specify which argument should be +used as the idempotency key. ```ts -import { makeIdempotent } from '@aws-lambda-powertools/idempotency'; -import { DynamoDBPersistenceLayer } from '@aws-lambda-powertools/idempotency/dynamodb'; -import type { Context, SQSEvent, SQSRecord } from 'aws-lambda'; +import {makeIdempotent} from '@aws-lambda-powertools/idempotency'; +import {DynamoDBPersistenceLayer} from '@aws-lambda-powertools/idempotency/dynamodb'; +import type {Context, SQSEvent, SQSRecord} from 'aws-lambda'; const persistenceStore = new DynamoDBPersistenceLayer({ - tableName: 'idempotencyTableName', + tableName: 'idempotencyTableName', }); const processingFunction = async (payload: SQSRecord, customerId: string): Promise => { - // your code goes here here + // your code goes here here }; const processIdempotently = makeIdempotent(processingFunction, { - persistenceStore, - // this tells the utility to use the second argument (`customerId`) as the idempotency key - dataIndexArgument: 1, + persistenceStore, + // this tells the utility to use the second argument (`customerId`) as the idempotency key + dataIndexArgument: 1, }); export const handler = async ( - event: SQSEvent, - _context: Context + event: SQSEvent, + _context: Context ): Promise => { - for (const record of event.Records) { - await processIdempotently(record, 'customer-123'); - } + for (const record of event.Records) { + await processIdempotently(record, 'customer-123'); + } }; ``` -Note that you can also specify a JMESPath expression in the Idempotency config object to select a subset of the event payload as the idempotency key. This is useful when dealing with payloads that contain timestamps or request ids. +Note that you can also specify a JMESPath expression in the Idempotency config object to select a subset of the event +payload as the idempotency key. This is useful when dealing with payloads that contain timestamps or request ids. ```ts -import { makeIdempotent, IdempotencyConfig } from '@aws-lambda-powertools/idempotency'; -import { DynamoDBPersistenceLayer } from '@aws-lambda-powertools/idempotency/dynamodb'; -import type { Context, APIGatewayProxyEvent } from 'aws-lambda'; +import {makeIdempotent, IdempotencyConfig} from '@aws-lambda-powertools/idempotency'; +import {DynamoDBPersistenceLayer} from '@aws-lambda-powertools/idempotency/dynamodb'; +import type {Context, APIGatewayProxyEvent} from 'aws-lambda'; const persistenceStore = new DynamoDBPersistenceLayer({ - tableName: 'idempotencyTableName', + tableName: 'idempotencyTableName', }); const myHandler = async ( - event: APIGatewayProxyEvent, - _context: Context + event: APIGatewayProxyEvent, + _context: Context ): Promise => { - // your code goes here here + // your code goes here here }; export const handler = makeIdempotent(myHandler, { - persistenceStore, - config: new IdempotencyConfig({ - eventKeyJmespath: 'requestContext.identity.user', - }), + persistenceStore, + config: new IdempotencyConfig({ + eventKeyJmespath: 'requestContext.identity.user', + }), }); ``` -Additionally, you can also use one of the [JMESPath built-in functions](https://docs.powertools.aws.dev/lambda/typescript/latest/utilities/jmespath/#built-in-jmespath-functions) like `powertools_json()` to decode keys and use parts of the payload as the idempotency key. +Additionally, you can also use one of +the [JMESPath built-in functions](https://docs.powertools.aws.dev/lambda/typescript/latest/utilities/jmespath/#built-in-jmespath-functions) +like `powertools_json()` to decode keys and use parts of the payload as the idempotency key. ```ts -import { makeIdempotent, IdempotencyConfig } from '@aws-lambda-powertools/idempotency'; -import { DynamoDBPersistenceLayer } from '@aws-lambda-powertools/idempotency/dynamodb'; -import type { Context, APIGatewayProxyEvent } from 'aws-lambda'; +import {makeIdempotent, IdempotencyConfig} from '@aws-lambda-powertools/idempotency'; +import {DynamoDBPersistenceLayer} from '@aws-lambda-powertools/idempotency/dynamodb'; +import type {Context, APIGatewayProxyEvent} from 'aws-lambda'; const persistenceStore = new DynamoDBPersistenceLayer({ - tableName: 'idempotencyTableName', + tableName: 'idempotencyTableName', }); const myHandler = async ( - event: APIGatewayProxyEvent, - _context: Context + event: APIGatewayProxyEvent, + _context: Context ): Promise => { - // your code goes here here + // your code goes here here }; export const handler = makeIdempotent(myHandler, { - persistenceStore, - config: new IdempotencyConfig({ - eventKeyJmespath: 'powertools_json(body).["user", "productId"]', - }), + persistenceStore, + config: new IdempotencyConfig({ + eventKeyJmespath: 'powertools_json(body).["user", "productId"]', + }), }); ``` @@ -184,23 +182,23 @@ Check the [docs](https://docs.powertools.aws.dev/lambda/typescript/latest/utilit You can make any function idempotent, and safe to retry, by decorating it using the `@idempotent` decorator. ```ts -import { idempotent } from '@aws-lambda-powertools/idempotency'; -import { LambdaInterface } from '@aws-lambda-powertools/commons'; -import { DynamoDBPersistenceLayer } from '@aws-lambda-powertools/idempotency/dynamodb'; -import type { Context, APIGatewayProxyEvent } from 'aws-lambda'; +import {idempotent} from '@aws-lambda-powertools/idempotency'; +import {LambdaInterface} from '@aws-lambda-powertools/commons'; +import {DynamoDBPersistenceLayer} from '@aws-lambda-powertools/idempotency/dynamodb'; +import type {Context, APIGatewayProxyEvent} from 'aws-lambda'; const persistenceStore = new DynamoDBPersistenceLayer({ - tableName: 'idempotencyTableName', + tableName: 'idempotencyTableName', }); class MyHandler extends LambdaInterface { - @idempotent({ persistenceStore: dynamoDBPersistenceLayer }) - public async handler( - event: APIGatewayProxyEvent, - context: Context - ): Promise { - // your code goes here here - } + @idempotent({persistenceStore: dynamoDBPersistenceLayer}) + public async handler( + event: APIGatewayProxyEvent, + context: Context + ): Promise { + // your code goes here here + } } const handlerClass = new MyHandler(); @@ -210,72 +208,77 @@ export const handler = handlerClass.handler.bind(handlerClass); Using the same decorator, you can also make any other arbitrary method idempotent. ```ts -import { idempotent } from '@aws-lambda-powertools/idempotency'; -import { LambdaInterface } from '@aws-lambda-powertools/commons'; -import { DynamoDBPersistenceLayer } from '@aws-lambda-powertools/idempotency/dynamodb'; -import type { Context } from 'aws-lambda'; +import {idempotent} from '@aws-lambda-powertools/idempotency'; +import {LambdaInterface} from '@aws-lambda-powertools/commons'; +import {DynamoDBPersistenceLayer} from '@aws-lambda-powertools/idempotency/dynamodb'; +import type {Context} from 'aws-lambda'; const persistenceStore = new DynamoDBPersistenceLayer({ - tableName: 'idempotencyTableName', + tableName: 'idempotencyTableName', }); class MyHandler extends LambdaInterface { - - public async handler( - event: unknown, - context: Context - ): Promise { - for(const record of event.Records) { - await this.processIdempotently(record); + + public async handler( + event: unknown, + context: Context + ): Promise { + for (const record of event.Records) { + await this.processIdempotently(record); + } + } + + @idempotent({persistenceStore: dynamoDBPersistenceLayer}) + private async process(record: unknown): Promise { + // process each code idempotently } - } - - @idempotent({ persistenceStore: dynamoDBPersistenceLayer }) - private async process(record: unknown): Promise { - // process each code idempotently - } } const handlerClass = new MyHandler(); export const handler = handlerClass.handler.bind(handlerClass); ``` -The decorator configuration options are identical with the ones of the `makeIdempotent` function. Check the [docs](https://docs.powertools.aws.dev/lambda/typescript/latest/utilities/idempotency/) for more examples. +The decorator configuration options are identical with the ones of the `makeIdempotent` function. Check +the [docs](https://docs.powertools.aws.dev/lambda/typescript/latest/utilities/idempotency/) for more examples. ### Middy middleware -If instead you use Middy, you can use the `makeHandlerIdempotent` middleware. When using the middleware your Lambda handler becomes idempotent. +If instead you use Middy, you can use the `makeHandlerIdempotent` middleware. When using the middleware your Lambda +handler becomes idempotent. -By default, the Idempotency utility will use the full event payload to create an hash and determine if a request is idempotent, and therefore it should not be retried. -When dealing with a more elaborate payload, where parts of the payload always change you should use the `IdempotencyConfig` object to instruct the utility to only use a portion of your payload. This is useful when dealing with payloads that contain timestamps or request ids. +By default, the Idempotency utility will use the full event payload to create an hash and determine if a request is +idempotent, and therefore it should not be retried. +When dealing with a more elaborate payload, where parts of the payload always change you should use +the `IdempotencyConfig` object to instruct the utility to only use a portion of your payload. This is useful when +dealing with payloads that contain timestamps or request ids. ```ts -import { IdempotencyConfig } from '@aws-lambda-powertools/idempotency'; -import { makeHandlerIdempotent } from '@aws-lambda-powertools/idempotency/middleware'; -import { DynamoDBPersistenceLayer } from '@aws-lambda-powertools/idempotency/dynamodb'; +import {IdempotencyConfig} from '@aws-lambda-powertools/idempotency'; +import {makeHandlerIdempotent} from '@aws-lambda-powertools/idempotency/middleware'; +import {DynamoDBPersistenceLayer} from '@aws-lambda-powertools/idempotency/dynamodb'; import middy from '@middy/core'; -import type { Context, APIGatewayProxyEvent } from 'aws-lambda'; +import type {Context, APIGatewayProxyEvent} from 'aws-lambda'; const persistenceStore = new DynamoDBPersistenceLayer({ - tableName: 'idempotencyTableName', + tableName: 'idempotencyTableName', }); const config = new IdempotencyConfig({ - hashFunction: 'md5', - useLocalCache: false, - expiresAfterSeconds: 3600, - throwOnNoIdempotencyKey: false, - eventKeyJmesPath: 'headers.idempotency-key', + hashFunction: 'md5', + useLocalCache: false, + expiresAfterSeconds: 3600, + throwOnNoIdempotencyKey: false, + eventKeyJmesPath: 'headers.idempotency-key', }); export const handler = middy( - async (_event: APIGatewayProxyEvent, _context: Context): Promise => { - // your code goes here here - } + async (_event: APIGatewayProxyEvent, _context: Context): Promise => { + // your code goes here here + } ).use( - makeHandlerIdempotent({ - config, - persistenceStore, - }) + makeHandlerIdempotent({ + config, + persistenceStore, + }) ); ``` @@ -283,19 +286,26 @@ Check the [docs](https://docs.powertools.aws.dev/lambda/typescript/latest/utilit ### DynamoDB persistence layer -You can use a DynamoDB Table to store the idempotency information. This enables you to keep track of the hash key, payload, status for progress, expiration, and much more. +You can use a DynamoDB Table to store the idempotency information. This enables you to keep track of the hash key, +payload, status for progress, expiration, and much more. You can customize most of the configuration options of the table, i.e the names of the attributes. -See the [API documentation](https://docs.powertools.aws.dev/lambda/typescript/latest/api/types/_aws_lambda_powertools_idempotency.types.DynamoDBPersistenceOptions.html) for more details. +See +the [API documentation](https://docs.powertools.aws.dev/lambda/typescript/latest/api/types/_aws_lambda_powertools_idempotency.types.DynamoDBPersistenceOptions.html) +for more details. ## Contribute -If you are interested in contributing to this project, please refer to our [Contributing Guidelines](https://github.com/aws-powertools/powertools-lambda-typescript/blob/main/CONTRIBUTING.md). +If you are interested in contributing to this project, please refer to +our [Contributing Guidelines](https://github.com/aws-powertools/powertools-lambda-typescript/blob/main/CONTRIBUTING.md). ## Roadmap The roadmap of Powertools for AWS Lambda (TypeScript) is driven by customers’ demand. -Help us prioritize upcoming functionalities or utilities by [upvoting existing RFCs and feature requests](https://github.com/aws-powertools/powertools-lambda-typescript/issues), or [creating new ones](https://github.com/aws-powertools/powertools-lambda-typescript/issues/new/choose), in this GitHub repository. +Help us prioritize upcoming functionalities or utilities +by [upvoting existing RFCs and feature requests](https://github.com/aws-powertools/powertools-lambda-typescript/issues), +or [creating new ones](https://github.com/aws-powertools/powertools-lambda-typescript/issues/new/choose), in this GitHub +repository. ## Connect @@ -306,7 +316,10 @@ Help us prioritize upcoming functionalities or utilities by [upvoting existing R ### Becoming a reference customer -Knowing which companies are using this library is important to help prioritize the project internally. If your company is using Powertools for AWS Lambda (TypeScript), you can request to have your name and logo added to the README file by raising a [Support Powertools for AWS Lambda (TypeScript) (become a reference)](https://s12d.com/become-reference-pt-ts) issue. +Knowing which companies are using this library is important to help prioritize the project internally. If your company +is using Powertools for AWS Lambda (TypeScript), you can request to have your name and logo added to the README file by +raising a [Support Powertools for AWS Lambda (TypeScript) (become a reference)](https://s12d.com/become-reference-pt-ts) +issue. The following companies, among others, use Powertools: @@ -328,11 +341,16 @@ The following companies, among others, use Powertools: ### Sharing your work -Share what you did with Powertools for AWS Lambda (TypeScript) 💞💞. Blog post, workshops, presentation, sample apps and others. Check out what the community has already shared about Powertools for AWS Lambda (TypeScript) [here](https://docs.powertools.aws.dev/lambda/typescript/latest/we_made_this). +Share what you did with Powertools for AWS Lambda (TypeScript) 💞💞. Blog post, workshops, presentation, sample apps and +others. Check out what the community has already shared about Powertools for AWS Lambda ( +TypeScript) [here](https://docs.powertools.aws.dev/lambda/typescript/latest/we_made_this). ### Using Lambda Layer -This helps us understand who uses Powertools for AWS Lambda (TypeScript) in a non-intrusive way, and helps us gain future investments for other Powertools for AWS Lambda languages. When [using Layers](https://docs.powertools.aws.dev/lambda/typescript/latest/#lambda-layer), you can add Powertools as a dev dependency to not impact the development process. +This helps us understand who uses Powertools for AWS Lambda (TypeScript) in a non-intrusive way, and helps us gain +future investments for other Powertools for AWS Lambda languages. +When [using Layers](https://docs.powertools.aws.dev/lambda/typescript/latest/#lambda-layer), you can add Powertools as a +dev dependency to not impact the development process. ## License From e28fc08f163f3cf09fa8de8754a49ba7effd4b27 Mon Sep 17 00:00:00 2001 From: Alexander Schueren Date: Tue, 13 Aug 2024 13:54:20 +0200 Subject: [PATCH 06/10] add testing section placeholder --- docs/utilities/idempotency.md | 283 +++++------------- .../samples/testingIdempotency.json | 30 -- .../idempotency/testingIdempotency.test.ts | 38 --- .../idempotency/testingIdempotency.ts | 24 -- package.json | 2 +- .../src/types/IdempotencyOptions.ts | 3 +- 6 files changed, 84 insertions(+), 296 deletions(-) delete mode 100644 examples/snippets/idempotency/samples/testingIdempotency.json delete mode 100644 examples/snippets/idempotency/testingIdempotency.test.ts delete mode 100644 examples/snippets/idempotency/testingIdempotency.ts diff --git a/docs/utilities/idempotency.md b/docs/utilities/idempotency.md index a9cf3a22fd..44bdda31ea 100644 --- a/docs/utilities/idempotency.md +++ b/docs/utilities/idempotency.md @@ -3,8 +3,7 @@ title: Idempotency description: Utility --- -The idempotency utility provides a simple solution to convert your Lambda functions into idempotent operations which are -safe to retry. +The idempotency utility provides a simple solution to convert your Lambda functions into idempotent operations which are safe to retry. ## Key features @@ -16,17 +15,13 @@ safe to retry. ## Terminology -The property of idempotency means that an operation does not cause additional side effects if it is called more than -once with the same input parameters. +The property of idempotency means that an operation does not cause additional side effects if it is called more than once with the same input parameters. -**Idempotent operations will return the same result when they are called multiple times with the same parameters**. This -makes idempotent operations safe to retry. +**Idempotent operations will return the same result when they are called multiple times with the same parameters**. This makes idempotent operations safe to retry. -**Idempotency key** is a hash representation of either the entire event or a specific configured subset of the event, -and invocation results are **JSON serialized** and stored in your persistence storage layer. +**Idempotency key** is a hash representation of either the entire event or a specific configured subset of the event, and invocation results are **JSON serialized** and stored in your persistence storage layer. -**Idempotency record** is the data representation of an idempotent request saved in your preferred storage layer. We use -it to coordinate whether a request is idempotent, whether it's still valid or expired based on timestamps, etc. +**Idempotency record** is the data representation of an idempotent request saved in your preferred storage layer. We use it to coordinate whether a request is idempotent, whether it's still valid or expired based on timestamps, etc.
```mermaid @@ -62,41 +57,32 @@ Install the library in your project npm i @aws-lambda-powertools/idempotency @aws-sdk/client-dynamodb @aws-sdk/lib-dynamodb ``` -While we support Amazon DynamoDB as a persistence layer out of the box, you need to bring your own AWS SDK for -JavaScript v3 DynamoDB client. +While we support Amazon DynamoDB as a persistence layer out of the box, you need to bring your own AWS SDK for JavaScript v3 DynamoDB client. ???+ note -This utility supports **[AWS SDK for JavaScript v3](https://docs.aws.amazon.com/AWSJavaScriptSDK/v3/latest/){target=" -_blank"} only**. If you are using the `nodejs18.x` runtime or newer, the AWS SDK for JavaScript v3 is already installed -and you can install only the utility. + This utility supports **[AWS SDK for JavaScript v3](https://docs.aws.amazon.com/AWSJavaScriptSDK/v3/latest/){target="_blank"} only**. If you are using the `nodejs18.x` runtime or newer, the AWS SDK for JavaScript v3 is already installed and you can install only the utility. ### IAM Permissions -Your Lambda function IAM Role must have `dynamodb:GetItem`, `dynamodb:PutItem`, `dynamodb:UpdateItem` -and `dynamodb:DeleteItem` IAM permissions before using this feature. If you're using one of our -examples: [AWS Serverless Application Model (SAM)](#required-resources) or [Terraform](#required-resources) the required -permissions are already included. +Your Lambda function IAM Role must have `dynamodb:GetItem`, `dynamodb:PutItem`, `dynamodb:UpdateItem` and `dynamodb:DeleteItem` IAM permissions before using this feature. If you're using one of our examples: [AWS Serverless Application Model (SAM)](#required-resources) or [Terraform](#required-resources) the required permissions are already included. ### Required resources -Before getting started, you need to create a persistent storage layer where the idempotency utility can store its -state - your lambda functions will need read and write access to it. +Before getting started, you need to create a persistent storage layer where the idempotency utility can store its state - your lambda functions will need read and write access to it. As of now, Amazon DynamoDB is the only supported persistent storage layer, so you'll need to create a table first. **Default table configuration** -If you're not [changing the default configuration for the DynamoDB persistence layer](#dynamodbpersistencelayer), this -is the expected default configuration: +If you're not [changing the default configuration for the DynamoDB persistence layer](#dynamodbpersistencelayer), this is the expected default configuration: | Configuration | Default value | Notes | -|--------------------|:--------------|----------------------------------------------------------------------------------------| +| ------------------ | :------------ | -------------------------------------------------------------------------------------- | | Partition key | `id` | The id of each idempotency record which a combination of `functionName#hashOfPayload`. | | TTL attribute name | `expiration` | This can only be configured after your table is created if you're using AWS Console. | ???+ tip "Tip: You can share a single state table for all functions" -You can reuse the same DynamoDB table to store idempotency state. We add the Lambda function name in addition to the -idempotency key as a hash key. + You can reuse the same DynamoDB table to store idempotency state. We add the Lambda function name in addition to the idempotency key as a hash key. === "AWS Cloud Development Kit (CDK) example" @@ -117,28 +103,20 @@ idempotency key as a hash key. ``` ???+ warning "Warning: Large responses with DynamoDB persistence layer" -When using this utility with DynamoDB, your function's responses must -be [smaller than 400KB](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Limits.html#limits-items) -{target="_blank"}. + When using this utility with DynamoDB, your function's responses must be [smaller than 400KB](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Limits.html#limits-items){target="_blank"}. Larger items cannot be written to DynamoDB and will cause exceptions. ???+ info "Info: DynamoDB" -Each function invocation will make only 1 request to DynamoDB by using -DynamoDB's [conditional expressions](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Expressions.ConditionExpressions.html) -{target="_blank"} to ensure that we don't overwrite existing records, -and [ReturnValuesOnConditionCheckFailure](https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_PutItem.html#DDB-PutItem-request-ReturnValuesOnConditionCheckFailure) -{target="_blank"} to return the record if it exists. -See [AWS Blog post on handling conditional write errors](https://aws.amazon.com/blogs/database/handle-conditional-write-errors-in-high-concurrency-scenarios-with-amazon-dynamodb/) -for more details. -For retried invocations, you will see 1WCU and 1RCU. -Review the [DynamoDB pricing documentation](https://aws.amazon.com/dynamodb/pricing/){target="_blank"} to estimate the -cost. + Each function invocation will make only 1 request to DynamoDB by using DynamoDB's [conditional expressions](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Expressions.ConditionExpressions.html){target="_blank"} to ensure that we don't overwrite existing records, + and [ReturnValuesOnConditionCheckFailure](https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_PutItem.html#DDB-PutItem-request-ReturnValuesOnConditionCheckFailure){target="_blank"} to return the record if it exists. + See [AWS Blog post on handling conditional write errors](https://aws.amazon.com/blogs/database/handle-conditional-write-errors-in-high-concurrency-scenarios-with-amazon-dynamodb/) for more details. + For retried invocations, you will see 1WCU and 1RCU. + Review the [DynamoDB pricing documentation](https://aws.amazon.com/dynamodb/pricing/){target="_blank"} to estimate the cost. ### MakeIdempotent function wrapper -You can quickly start by initializing the `DynamoDBPersistenceLayer` class and using it with the `makeIdempotent` -function wrapper on your Lambda handler. +You can quickly start by initializing the `DynamoDBPersistenceLayer` class and using it with the `makeIdempotent` function wrapper on your Lambda handler. === "index.ts" @@ -152,26 +130,19 @@ function wrapper on your Lambda handler. --8<-- "examples/snippets/idempotency/types.ts:3:16" ``` -After processing this request successfully, a second request containing the exact same payload above will now return the -same response, ensuring our customer isn't charged twice. +After processing this request successfully, a second request containing the exact same payload above will now return the same response, ensuring our customer isn't charged twice. ???+ note -In this example, the entire Lambda handler is treated as a single idempotent operation. If your Lambda handler can cause -multiple side effects, or you're only interested in making a specific logic idempotent, use the `makeIdempotent` -high-order function only on the function that needs to be idempotent. + In this example, the entire Lambda handler is treated as a single idempotent operation. If your Lambda handler can cause multiple side effects, or you're only interested in making a specific logic idempotent, use the `makeIdempotent` high-order function only on the function that needs to be idempotent. See [Choosing a payload subset for idempotency](#choosing-a-payload-subset-for-idempotency) for more elaborate use cases. -You can also use the `makeIdempotent` function wrapper on any method that returns a response to make it idempotent. This -is useful when you want to make a specific logic idempotent, for example when your Lambda handler performs multiple side -effects and you only want to make a specific one idempotent. +You can also use the `makeIdempotent` function wrapper on any method that returns a response to make it idempotent. This is useful when you want to make a specific logic idempotent, for example when your Lambda handler performs multiple side effects and you only want to make a specific one idempotent. ???+ warning "Limitation" -Make sure to return a JSON serializable response from your function, otherwise you'll get an error. + Make sure to return a JSON serializable response from your function, otherwise you'll get an error. -When using `makeIdempotent` on arbitrary functions, you can tell us which argument in your function signature has the -data we should use via **`dataIndexArgument`**. If you don't specify this argument, we'll use the first argument in the -function signature. +When using `makeIdempotent` on arbitrary functions, you can tell us which argument in your function signature has the data we should use via **`dataIndexArgument`**. If you don't specify this argument, we'll use the first argument in the function signature. === "index.ts" @@ -185,20 +156,14 @@ function signature. --8<-- "examples/snippets/idempotency/types.ts:3:16" ``` -The function this example has two arguments, note that while wrapping it with the `makeIdempotent` high-order function, -we specify the `dataIndexArgument` as `1` to tell the decorator that the second argument is the one with the data we -should use to make the function idempotent. Remember that arguments are zero-indexed, so the first argument is `0`, the -second is `1`, etc. +The function this example has two arguments, note that while wrapping it with the `makeIdempotent` high-order function, we specify the `dataIndexArgument` as `1` to tell the decorator that the second argument is the one with the data we should use to make the function idempotent. Remember that arguments are zero-indexed, so the first argument is `0`, the second is `1`, etc. ### Idempotent Decorator -You can also use the `@idempotent` decorator to make your Lambda handler idempotent, similar to the `makeIdempotent` -function wrapper. +You can also use the `@idempotent` decorator to make your Lambda handler idempotent, similar to the `makeIdempotent` function wrapper. !!! info -The class method decorators in this project follow the experimental implementation enabled via -the [`experimentalDecorators` compiler option](https://www.typescriptlang.org/tsconfig#experimentalDecorators) in -TypeScript. + The class method decorators in this project follow the experimental implementation enabled via the [`experimentalDecorators` compiler option](https://www.typescriptlang.org/tsconfig#experimentalDecorators) in TypeScript. Additionally, they are implemented to decorate async methods. When decorating a synchronous one, the decorator replaces its implementation with an async one causing the caller to have to `await` the now decorated method. @@ -216,26 +181,18 @@ TypeScript. --8<-- "examples/snippets/idempotency/types.ts" ``` -You can use the decorator on your Lambda handler or on any function that returns a response to make it idempotent. This -is useful when you want to make a specific logic idempotent, for example when your Lambda handler performs multiple side -effects and you only want to make a specific one idempotent. -The configuration options for the `@idempotent` decorator are the same as the ones for the `makeIdempotent` function -wrapper. +You can use the decorator on your Lambda handler or on any function that returns a response to make it idempotent. This is useful when you want to make a specific logic idempotent, for example when your Lambda handler performs multiple side effects and you only want to make a specific one idempotent. +The configuration options for the `@idempotent` decorator are the same as the ones for the `makeIdempotent` function wrapper. ### MakeHandlerIdempotent Middy middleware !!! tip "A note about Middy" -We guarantee support for both Middy.js `v4.x` & `v5.x` with the latter being available only if you are using ES modules. -Check their docs to learn more about [Middy and its middleware stack](https://middy.js.org/docs/intro/getting-started) -{target="_blank"} as well -as [best practices when working with Powertools](https://middy.js.org/docs/integrations/lambda-powertools#best-practices) -{target="_blank"}. + We guarantee support for both Middy.js `v4.x` & `v5.x` with the latter being available only if you are using ES modules. + Check their docs to learn more about [Middy and its middleware stack](https://middy.js.org/docs/intro/getting-started){target="_blank"} as well as [best practices when working with Powertools](https://middy.js.org/docs/integrations/lambda-powertools#best-practices){target="_blank"}. -If you are using [Middy.js](https://middy.js.org){target="_blank"} as your middleware engine, you can use -the `makeHandlerIdempotent` middleware to make your Lambda handler idempotent. +If you are using [Middy.js](https://middy.js.org){target="_blank"} as your middleware engine, you can use the `makeHandlerIdempotent` middleware to make your Lambda handler idempotent. -Similar to the `makeIdempotent` function wrapper, you can quickly make your Lambda handler idempotent by initializing -the `DynamoDBPersistenceLayer` class and using it with the `makeHandlerIdempotent` middleware. +Similar to the `makeIdempotent` function wrapper, you can quickly make your Lambda handler idempotent by initializing the `DynamoDBPersistenceLayer` class and using it with the `makeHandlerIdempotent` middleware. === "index.ts" @@ -251,25 +208,18 @@ the `DynamoDBPersistenceLayer` class and using it with the `makeHandlerIdempoten ### Choosing a payload subset for idempotency -Use [`IdempotencyConfig`](#customizing-the-default-behavior) to instruct the idempotent decorator to only use a portion -of your payload to verify whether a request is idempotent, and therefore it should not be retried. When dealing with a -more elaborate payload, where parts of the payload always change, you should use the **`eventKeyJmesPath`** parameter. +Use [`IdempotencyConfig`](#customizing-the-default-behavior) to instruct the idempotent decorator to only use a portion of your payload to verify whether a request is idempotent, and therefore it should not be retried. When dealing with a more elaborate payload, where parts of the payload always change, you should use the **`eventKeyJmesPath`** parameter. **Payment scenario** -In this example, we have a Lambda handler that creates a payment for a user subscribing to a product. We want to ensure -that we don't accidentally charge our customer by subscribing them more than once. +In this example, we have a Lambda handler that creates a payment for a user subscribing to a product. We want to ensure that we don't accidentally charge our customer by subscribing them more than once. -Imagine the function executes successfully, but the client never receives the response due to a connection issue. It is -safe to retry in this instance, as the idempotent decorator will return a previously saved response. +Imagine the function executes successfully, but the client never receives the response due to a connection issue. It is safe to retry in this instance, as the idempotent decorator will return a previously saved response. -**What we want here** is to instruct Idempotency to use the `user` and `productId` fields from our incoming payload as -our idempotency key. If we were to treat the entire request as our idempotency key, a simple HTTP header or timestamp -change would cause our customer to be charged twice. +**What we want here** is to instruct Idempotency to use the `user` and `productId` fields from our incoming payload as our idempotency key. If we were to treat the entire request as our idempotency key, a simple HTTP header or timestamp change would cause our customer to be charged twice. ???+ warning "Deserializing JSON strings in payloads for increased accuracy." -The payload extracted by the `eventKeyJmesPath` is treated as a string by default. This means there could be differences -in whitespace even when the JSON payload itself is identical. + The payload extracted by the `eventKeyJmesPath` is treated as a string by default. This means there could be differences in whitespace even when the JSON payload itself is identical. To alter this behaviour, we can use the [JMESPath built-in function `powertools_json()`](jmespath.md#powertools_json-function) to treat the payload as a JSON object rather than a string. @@ -293,23 +243,16 @@ in whitespace even when the JSON payload itself is identical. ### Lambda timeouts -To prevent against extended failed retries when -a [Lambda function times out](https://aws.amazon.com/premiumsupport/knowledge-center/lambda-verify-invocation-timeouts/), -Powertools for AWS Lambda calculates and includes the remaining invocation available time as part of the idempotency -record. -This is automatically done when you wrap your Lambda handler with the [makeIdempotent](#makeidempotent-function-wrapper) -function wrapper, or use the [`makeHandlerIdempotent`](#makehandleridempotent-middy-middleware) Middy middleware. +To prevent against extended failed retries when a [Lambda function times out](https://aws.amazon.com/premiumsupport/knowledge-center/lambda-verify-invocation-timeouts/), Powertools for AWS Lambda calculates and includes the remaining invocation available time as part of the idempotency record. +This is automatically done when you wrap your Lambda handler with the [makeIdempotent](#makeidempotent-function-wrapper) function wrapper, or use the [`makeHandlerIdempotent`](#makehandleridempotent-middy-middleware) Middy middleware. ???+ example -If a second invocation happens **after** this timestamp, and the record is marked as `INPROGRESS`, we will execute the -invocation again as if it was in the `EXPIRED` state (e.g, `expire_seconds` field elapsed). + If a second invocation happens **after** this timestamp, and the record is marked as `INPROGRESS`, we will execute the invocation again as if it was in the `EXPIRED` state (e.g, `expire_seconds` field elapsed). This means that if an invocation expired during execution, it will be quickly executed again on the next retry. ???+ important -If you are only using the [makeIdempotent function wrapper](#makeidempotent-function-wrapper) to guard isolated parts of -your code outside of your handler, you must use `registerLambdaContext` available in -the [idempotency config object](#customizing-the-default-behavior) to benefit from this protection. + If you are only using the [makeIdempotent function wrapper](#makeidempotent-function-wrapper) to guard isolated parts of your code outside of your handler, you must use `registerLambdaContext` available in the [idempotency config object](#customizing-the-default-behavior) to benefit from this protection. Here is an example on how you register the Lambda context in your handler: @@ -321,10 +264,8 @@ Here is an example on how you register the Lambda context in your handler: ### Handling exceptions -If you are making on your entire Lambda handler idempotent, any unhandled exceptions that are raised during the code -execution will cause **the record in the persistence layer to be deleted**. -This means that new invocations will execute your code again despite having the same payload. If you don't want the -record to be deleted, you need to catch exceptions within the idempotent function and return a successful response. +If you are making on your entire Lambda handler idempotent, any unhandled exceptions that are raised during the code execution will cause **the record in the persistence layer to be deleted**. +This means that new invocations will execute your code again despite having the same payload. If you don't want the record to be deleted, you need to catch exceptions within the idempotent function and return a successful response.
```mermaid @@ -345,13 +286,9 @@ sequenceDiagram Idempotent sequence exception
-If you are using `makeIdempotent` on any other function, any unhandled exceptions that are thrown _inside_ the wrapped -function will cause the record in the persistence layer to be deleted, and allow the function to be executed again if -retried. +If you are using `makeIdempotent` on any other function, any unhandled exceptions that are thrown _inside_ the wrapped function will cause the record in the persistence layer to be deleted, and allow the function to be executed again if retried. -If an error is thrown _outside_ the scope of the decorated function and after your function has been called, the -persistent record will not be affected. In this case, idempotency will be maintained for your decorated function. -Example: +If an error is thrown _outside_ the scope of the decorated function and after your function has been called, the persistent record will not be affected. In this case, idempotency will be maintained for your decorated function. Example: === "Handling exceptions" @@ -360,7 +297,7 @@ Example: ``` ???+ warning -**We will throw `IdempotencyPersistenceLayerError`** if any of the calls to the persistence layer fail unexpectedly. + **We will throw `IdempotencyPersistenceLayerError`** if any of the calls to the persistence layer fail unexpectedly. As this happens outside the scope of your decorated function, you are not able to catch it when making your Lambda handler idempotent. @@ -570,8 +507,7 @@ sequenceDiagram #### DynamoDBPersistenceLayer -This persistence layer is built-in, and you can either use an existing DynamoDB table or create a new one dedicated for -idempotency state (recommended). +This persistence layer is built-in, and you can either use an existing DynamoDB table or create a new one dedicated for idempotency state (recommended). === "Customizing DynamoDBPersistenceLayer to suit your table structure" @@ -579,11 +515,10 @@ idempotency state (recommended). --8<-- "examples/snippets/idempotency/customizePersistenceLayer.ts" ``` -When using DynamoDB as a persistence layer, you can alter the attribute names by passing these parameters when -initializing the persistence layer: +When using DynamoDB as a persistence layer, you can alter the attribute names by passing these parameters when initializing the persistence layer: | Parameter | Required | Default | Description | -|--------------------------|--------------------|--------------------------------------|----------------------------------------------------------------------------------------------------------| +| ------------------------ | ------------------ | ------------------------------------ | -------------------------------------------------------------------------------------------------------- | | **tableName** | :heavy_check_mark: | | Table name to store state | | **keyAttr** | | `id` | Partition key of the table. Hashed representation of the payload (unless **sort_key_attr** is specified) | | **expiryAttr** | | `expiration` | Unix timestamp of when record expires | @@ -596,11 +531,10 @@ initializing the persistence layer: ### Customizing the default behavior -Idempotent decorator can be further configured with **`IdempotencyConfig`** as seen in the previous examples. These are -the available options for further configuration +Idempotent decorator can be further configured with **`IdempotencyConfig`** as seen in the previous examples. These are the available options for further configuration | Parameter | Default | Description | -|-------------------------------|---------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| ----------------------------- | ------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | | **eventKeyJmespath** | `''` | JMESPath expression to extract the idempotency key from the event | | **payloadValidationJmespath** | `''` | JMESPath expression to validate whether certain parameters have changed in the event while the event payload | | **throwOnNoIdempotencyKey** | `false` | Throw an error if no idempotency key was found in the request | @@ -611,23 +545,19 @@ the available options for further configuration ### Handling concurrent executions with the same payload -This utility will throw an **`IdempotencyAlreadyInProgressError`** error if you receive **multiple invocations with the -same payload while the first invocation hasn't completed yet**. +This utility will throw an **`IdempotencyAlreadyInProgressError`** error if you receive **multiple invocations with the same payload while the first invocation hasn't completed yet**. ???+ info -If you receive `IdempotencyAlreadyInProgressError`, you can safely retry the operation. + If you receive `IdempotencyAlreadyInProgressError`, you can safely retry the operation. -This is a locking mechanism for correctness. Since we don't know the result from the first invocation yet, we can't -safely allow another concurrent execution. +This is a locking mechanism for correctness. Since we don't know the result from the first invocation yet, we can't safely allow another concurrent execution. ### Using in-memory cache -**By default, in-memory local caching is disabled**, since we don't know how much memory you consume per invocation -compared to the maximum configured in your Lambda function. +**By default, in-memory local caching is disabled**, since we don't know how much memory you consume per invocation compared to the maximum configured in your Lambda function. ???+ note "Note: This in-memory cache is local to each Lambda execution environment" -This means it will be effective in cases where your function's concurrency is low in comparison to the number of "retry" -invocations with the same payload, because cache might be empty. + This means it will be effective in cases where your function's concurrency is low in comparison to the number of "retry" invocations with the same payload, because cache might be empty. You can enable in-memory caching with the **`useLocalCache`** parameter: @@ -637,15 +567,13 @@ You can enable in-memory caching with the **`useLocalCache`** parameter: --8<-- "examples/snippets/idempotency/workingWithLocalCache.ts" ``` -When enabled, the default is to cache a maximum of 256 records in each Lambda execution environment - You can change it -with the **`maxLocalCacheSize`** parameter. +When enabled, the default is to cache a maximum of 256 records in each Lambda execution environment - You can change it with the **`maxLocalCacheSize`** parameter. ### Expiring idempotency records !!! note "By default, we expire idempotency records after **an hour** (3600 seconds)." -In most cases, it is not desirable to store the idempotency records forever. Rather, you want to guarantee that the same -payload won't be executed within a period of time. +In most cases, it is not desirable to store the idempotency records forever. Rather, you want to guarantee that the same payload won't be executed within a period of time. You can change this window with the **`expiresAfterSeconds`** parameter: @@ -655,12 +583,10 @@ You can change this window with the **`expiresAfterSeconds`** parameter: --8<-- "examples/snippets/idempotency/workingWithRecordExpiration.ts" ``` -This will mark any records older than 5 minutes as expired, -and [your function will be executed as normal if it is invoked with a matching payload](#expired-idempotency-records). +This will mark any records older than 5 minutes as expired, and [your function will be executed as normal if it is invoked with a matching payload](#expired-idempotency-records). ???+ important "Idempotency record expiration vs DynamoDB time-to-live (TTL)" -[DynamoDB TTL is a feature](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/howitworks-ttl.html) -{target="_blank"} to remove items after a certain period of time, it may occur within 48 hours of expiration. + [DynamoDB TTL is a feature](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/howitworks-ttl.html){target="_blank"} to remove items after a certain period of time, it may occur within 48 hours of expiration. We don't rely on DynamoDB or any persistence storage layer to determine whether a record is expired to avoid eventual inconsistency states. @@ -672,16 +598,12 @@ and [your function will be executed as normal if it is invoked with a matching p ### Payload validation -???+ question "Question: What if your function is invoked with the same payload except some outer parameters have -changed?" -Example: A payment transaction for a given productID was requested twice for the same customer, **however the amount to -be paid has changed in the second transaction**. +???+ question "Question: What if your function is invoked with the same payload except some outer parameters have changed?" + Example: A payment transaction for a given productID was requested twice for the same customer, **however the amount to be paid has changed in the second transaction**. -By default, we will return the same result as it returned before, however in this instance it may be misleading; we -provide a fail fast payload validation to address this edge case. +By default, we will return the same result as it returned before, however in this instance it may be misleading; we provide a fail fast payload validation to address this edge case. -With **`payloadValidationJmesPath`**, you can provide an additional JMESPath expression to specify which part of the -event body should be validated against previous idempotent invocations +With **`payloadValidationJmesPath`**, you can provide an additional JMESPath expression to specify which part of the event body should be validated against previous idempotent invocations === "Payload validation" @@ -689,29 +611,23 @@ event body should be validated against previous idempotent invocations --8<-- "examples/snippets/idempotency/workingWithPayloadValidation.ts" ``` -In this example, the **`userId`** and **`productId`** keys are used as the payload to generate the idempotency key, as -per **`eventKeyJmespath`** parameter. +In this example, the **`userId`** and **`productId`** keys are used as the payload to generate the idempotency key, as per **`eventKeyJmespath`** parameter. ???+ note -If we try to send the same request but with a different amount, we will raise **`IdempotencyValidationError`**. + If we try to send the same request but with a different amount, we will raise **`IdempotencyValidationError`**. -Without payload validation, we would have returned the same result as we did for the initial request. Since we're also -returning an amount in the response, this could be quite confusing for the client. +Without payload validation, we would have returned the same result as we did for the initial request. Since we're also returning an amount in the response, this could be quite confusing for the client. -By using **`payloadValidationJmesPath="amount"`**, we prevent this potentially confusing behavior and instead throw an -error. +By using **`payloadValidationJmesPath="amount"`**, we prevent this potentially confusing behavior and instead throw an error. ### Making idempotency key required If you want to enforce that an idempotency key is required, you can set **`throwOnNoIdempotencyKey`** to `true`. -This means that we will raise **`IdempotencyKeyError`** if the evaluation of **`eventKeyJmesPath`** results in an empty -subset. +This means that we will raise **`IdempotencyKeyError`** if the evaluation of **`eventKeyJmesPath`** results in an empty subset. ???+ warning -To prevent errors, transactions will not be treated as idempotent if **`throwOnNoIdempotencyKey`** is set to `false` and -the evaluation of **`eventKeyJmesPath`** is an empty result. Therefore, no data will be fetched, stored, or deleted in -the idempotency storage layer. + To prevent errors, transactions will not be treated as idempotent if **`throwOnNoIdempotencyKey`** is set to `false` and the evaluation of **`eventKeyJmesPath`** is an empty result. Therefore, no data will be fetched, stored, or deleted in the idempotency storage layer. === "Idempotency key required" @@ -734,14 +650,11 @@ the idempotency storage layer. ### Batch integration You can easily integrate with [Batch](batch.md) utility by using idempotency wrapper around your processing function. -This ensures that you process each record in an idempotent manner, and guard against -a [Lambda timeout](#lambda-timeouts) idempotent situation. +This ensures that you process each record in an idempotent manner, and guard against a [Lambda timeout](#lambda-timeouts) idempotent situation. ???+ "Choosing a unique batch record attribute" -In this example, we choose `messageId` as our idempotency key since we know it'll be unique. -Depending on your use case, it might be more -accurate [to choose another field](#choosing-a-payload-subset-for-idempotency) your producer intentionally set to define -uniqueness. + In this example, we choose `messageId` as our idempotency key since we know it'll be unique. + Depending on your use case, it might be more accurate [to choose another field](#choosing-a-payload-subset-for-idempotency) your producer intentionally set to define uniqueness. === "Integration with batch processor" @@ -757,9 +670,7 @@ uniqueness. ### Customizing AWS SDK configuration -The **`clientConfig`** and **`awsSdkV3Client`** parameters enable you to pass in custom configurations or your -own [DynamoDBClient](https://docs.aws.amazon.com/AWSJavaScriptSDK/v3/latest/client/dynamodb/){target="_blank"} when -constructing the persistence store. +The **`clientConfig`** and **`awsSdkV3Client`** parameters enable you to pass in custom configurations or your own [DynamoDBClient](https://docs.aws.amazon.com/AWSJavaScriptSDK/v3/latest/client/dynamodb/){target="_blank"} when constructing the persistence store. === "Passing specific configuration" @@ -775,11 +686,9 @@ constructing the persistence store. ### Using a DynamoDB table with a composite primary key -When using a composite primary key table (hash+range key), use `sortKeyAttr` parameter when initializing your -persistence layer. +When using a composite primary key table (hash+range key), use `sortKeyAttr` parameter when initializing your persistence layer. -With this setting, we will save the idempotency key in the sort key instead of the primary key. By default, the primary -key will now be set to `idempotency#{LAMBDA_FUNCTION_NAME}`. +With this setting, we will save the idempotency key in the sort key instead of the primary key. By default, the primary key will now be set to `idempotency#{LAMBDA_FUNCTION_NAME}`. You can optionally set a static value for the partition key using the `staticPkValue` parameter. @@ -792,7 +701,7 @@ You can optionally set a static value for the partition key using the `staticPkV The example function above would cause data to be stored in DynamoDB like this: | id | sort_key | expiration | status | data | -|------------------------------|----------------------------------|------------|-------------|------------------------------------------------------------------| +| ---------------------------- | -------------------------------- | ---------- | ----------- | ---------------------------------------------------------------- | | idempotency#MyLambdaFunction | 1e956ef7da78d0cb890be999aecc0c9e | 1636549553 | COMPLETED | {"paymentId": "12345, "message": "success", "statusCode": 200} | | idempotency#MyLambdaFunction | 2b2cdb5f86361e97b4383087c1ffdf27 | 1636549571 | COMPLETED | {"paymentId": "527212", "message": "success", "statusCode": 200} | | idempotency#MyLambdaFunction | f091d2527ad1c78f05d54cc3f363be80 | 1636549585 | IN_PROGRESS | | @@ -801,13 +710,10 @@ The example function above would cause data to be stored in DynamoDB like this: This utility provides an abstract base class (ABC), so that you can implement your choice of persistent storage layer. -You can create your own persistent store from scratch by inheriting the `BasePersistenceLayer` class, and -implementing `_getRecord()`, `_putRecord()`, `_updateRecord()` and `_deleteRecord()`. +You can create your own persistent store from scratch by inheriting the `BasePersistenceLayer` class, and implementing `_getRecord()`, `_putRecord()`, `_updateRecord()` and `_deleteRecord()`. -* `_getRecord()` – Retrieves an item from the persistence store using an idempotency key and returns it as - a `IdempotencyRecord` instance. -* `_putRecord()` – Adds a `IdempotencyRecord` to the persistence store if it doesn't already exist with that key. Throws - an `IdempotencyItemAlreadyExistsError` error if a non-expired entry already exists. +* `_getRecord()` – Retrieves an item from the persistence store using an idempotency key and returns it as a `IdempotencyRecord` instance. +* `_putRecord()` – Adds a `IdempotencyRecord` to the persistence store if it doesn't already exist with that key. Throws an `IdempotencyItemAlreadyExistsError` error if a non-expired entry already exists. * `_updateRecord()` – Updates an item in the persistence store. * `_deleteRecord()` – Removes an item from the persistence store. @@ -832,37 +738,12 @@ Below an example implementation of a custom persistence layer backed by a generi ``` ???+ danger -Pay attention to the documentation for each - you may need to perform additional checks inside these methods to ensure -the idempotency guarantees remain intact. + Pay attention to the documentation for each - you may need to perform additional checks inside these methods to ensure the idempotency guarantees remain intact. For example, the `_putRecord()` method needs to throw an error if a non-expired record already exists in the data store with a matching key. ## Testing your code -When testing idempotency, you can verify that your function stores the correct data in the persistence layer. -In this example, we send two different requests with the same payload. - -=== "index.ts" - - ```typescript - --8<-- "examples/snippets/idempotency/testingIdempotency.ts" - ``` - -=== "index.test.ts" - - ```typescript - --8<-- "examples/snippets/idempotency/testingIdempotency.test.ts" - ``` - -=== "Sample event" - - ```json - --8<-- "examples/snippets/idempotency/samples/testingIdempotency.json" - ``` - -You can also check how -we [tested idempotency in our end to end tests](https://github.com/aws-powertools/powertools-lambda-typescript/tree/main/packages/idempotency/tests/e2e). - ## Extra resources If you're interested in a deep dive on how Amazon uses idempotency when building our APIs, check out diff --git a/examples/snippets/idempotency/samples/testingIdempotency.json b/examples/snippets/idempotency/samples/testingIdempotency.json deleted file mode 100644 index 158b3a874f..0000000000 --- a/examples/snippets/idempotency/samples/testingIdempotency.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "version": "2.0", - "routeKey": "ANY /createpayment", - "rawPath": "/createpayment", - "rawQueryString": "", - "headers": { - "Header1": "value1", - "X-Idempotency-Key": "abcdefg" - }, - "requestContext": { - "accountId": "123456789012", - "apiId": "api-id", - "domainName": "id.execute-api.us-east-1.amazonaws.com", - "domainPrefix": "id", - "http": { - "method": "POST", - "path": "/createpayment", - "protocol": "HTTP/1.1", - "sourceIp": "ip", - "userAgent": "agent" - }, - "requestId": "id", - "routeKey": "ANY /createpayment", - "stage": "$default", - "time": "10/Feb/2021:13:40:43 +0000", - "timeEpoch": 1612964443723 - }, - "body": "{\"user\":\"xyz\",\"productId\":\"123456789\"}", - "isBase64Encoded": false -} diff --git a/examples/snippets/idempotency/testingIdempotency.test.ts b/examples/snippets/idempotency/testingIdempotency.test.ts deleted file mode 100644 index 499e08b004..0000000000 --- a/examples/snippets/idempotency/testingIdempotency.test.ts +++ /dev/null @@ -1,38 +0,0 @@ -import type { Context } from 'aws-lambda'; -import { DocumentClient } from 'aws-sdk/clients/dynamodb'; -import * as API_GATEWAY_EXAMPLE_EVENT from './samples/testingIdempotency.json'; -import { idempotentHandler } from './testingIdempotency'; - -describe('Idempotent Handler', () => { - const ddb = new DocumentClient({}); - - it('should return the same response for the same request', async () => { - // given - const context = {} as Context; - - const firstRequest = API_GATEWAY_EXAMPLE_EVENT; - - // modify time field to simulate a different request - const secondRequest = { - ...API_GATEWAY_EXAMPLE_EVENT, - requestContext: { - ...API_GATEWAY_EXAMPLE_EVENT.requestContext, - time: 1612964493723, - }, - }; - - // when - const firstResponse = await idempotentHandler(firstRequest, context); - const secondResponse = await idempotentHandler(secondRequest, context); - // then - expect(firstResponse).toEqual(secondResponse); - // check if we only have one item in the table - const idempotencyRecords = await ddb - .scan({ TableName: 'idempotency-store' }) - .promise(); - - expect(idempotencyRecords.Items).toHaveLength(1); - expect(idempotencyRecords.Items?.[0].status).toEqual('COMPLETED'); - expect(idempotencyRecords.Items?.[0].data).toEqual(firstResponse); - }); -}); diff --git a/examples/snippets/idempotency/testingIdempotency.ts b/examples/snippets/idempotency/testingIdempotency.ts deleted file mode 100644 index 02c7a54c48..0000000000 --- a/examples/snippets/idempotency/testingIdempotency.ts +++ /dev/null @@ -1,24 +0,0 @@ -import { - IdempotencyConfig, - makeIdempotent, -} from '@aws-lambda-powertools/idempotency'; -import { DynamoDBPersistenceLayer } from '@aws-lambda-powertools/idempotency/dynamodb'; - -const idempotencyConfig = new IdempotencyConfig({}); -const persistenceStore = new DynamoDBPersistenceLayer({ - tableName: 'idempotency-store', -}); - -const handler = async (event: unknown, context: unknown) => { - return { - statusCode: 200, - body: JSON.stringify({ message: 'Success', event: event }), - }; -}; - -const idempotentHandler = makeIdempotent(handler, { - config: idempotencyConfig, - persistenceStore: persistenceStore, -}); - -export { idempotentHandler }; diff --git a/package.json b/package.json index 9b2b5231c9..f172f5304a 100644 --- a/package.json +++ b/package.json @@ -33,7 +33,7 @@ "docs-generateApiDoc": "typedoc .", "docs-runLocalApiDoc": "npx live-server api", "postpublish": "git restore .", - "lint:markdown": "markdownlint-cli2 '**/*.md' '#node_modules' '#**/*/node_modules' '#docs/changelog.md' '#LICENSE.md' '#.github' '#**/*/CHANGELOG.md' '#examples/app/README.md' '#packages/commons/README.md' '#packages/idempotency/README.md' '#packages/jmespath/README.md' '#packages/logger/README.md' '#packages/metrics/README.md' '#packages/parameters/README.md' '#packages/parser/README.md' '#packages/tracer/README.md'" + "lint:markdown": "markdownlint-cli2 '**/*.md' '#node_modules' '#**/*/node_modules' '#docs/changelog.md' '#LICENSE.md' '#.github' '#**/*/CHANGELOG.md' '#examples/app/README.md' '#packages/commons/README.md' '#packages/jmespath/README.md' '#packages/logger/README.md' '#packages/metrics/README.md' '#packages/parameters/README.md' '#packages/parser/README.md' '#packages/tracer/README.md'" }, "repository": { "type": "git", diff --git a/packages/idempotency/src/types/IdempotencyOptions.ts b/packages/idempotency/src/types/IdempotencyOptions.ts index 1b07b11218..755e2edcac 100644 --- a/packages/idempotency/src/types/IdempotencyOptions.ts +++ b/packages/idempotency/src/types/IdempotencyOptions.ts @@ -110,8 +110,7 @@ type ItempotentFunctionOptions> = T[1] extends Context * @internal * Options to configure the behavior of the idempotency logic. * - * This is an internal type that is used by the Idempotency utility to - * configure. + * This is an internal type that is used for configuration. */ type IdempotencyHandlerOptions = { /** From 74c4edf4c2af04b5395a44a227d3ca902f962379 Mon Sep 17 00:00:00 2001 From: Alexander Schueren Date: Tue, 13 Aug 2024 14:40:20 +0200 Subject: [PATCH 07/10] add testing section --- docs/utilities/idempotency.md | 22 ++++++++++ .../workingWithLocalDynamoDB.test.ts | 44 +++++++++++++++++++ .../idempotency/workingWithLocalDynamoDB.ts | 23 ++++++++++ 3 files changed, 89 insertions(+) create mode 100644 examples/snippets/idempotency/workingWithLocalDynamoDB.test.ts create mode 100644 examples/snippets/idempotency/workingWithLocalDynamoDB.ts diff --git a/docs/utilities/idempotency.md b/docs/utilities/idempotency.md index 44bdda31ea..552fb4bcec 100644 --- a/docs/utilities/idempotency.md +++ b/docs/utilities/idempotency.md @@ -744,6 +744,28 @@ Below an example implementation of a custom persistence layer backed by a generi ## Testing your code +The idempotency utility provides several routes to test your code. + +### Disabling the idempotency utility + +When testing your code, you may wish to disable the idempotency logic altogether and focus on testing your business logic. To do this, you can set the environment variable POWERTOOLS_IDEMPOTENCY_DISABLED with a truthy value. + +### Testing with local DynamoDB + +When testing your Lambda function locally, you can use a local DynamoDB instance to test the idempotency feature. You can use [DynamoDB Local](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/DynamoDBLocal.DownloadingAndRunning.html) or [LocalStack](https://localstack.cloud/){target="_blank"}. + +=== "handler.test.ts" + + ```typescript hl_lines="7-9" + --8<-- "examples/snippets/idempotency/workingWithLocalDynamoDB.test.ts" + ``` + +=== "handler.ts" + + ```typescript hl_lines="7-9" + --8<-- "examples/snippets/idempotency/workingWithLocalDynamoDB.ts" + ``` + ## Extra resources If you're interested in a deep dive on how Amazon uses idempotency when building our APIs, check out diff --git a/examples/snippets/idempotency/workingWithLocalDynamoDB.test.ts b/examples/snippets/idempotency/workingWithLocalDynamoDB.test.ts new file mode 100644 index 0000000000..adf4f18847 --- /dev/null +++ b/examples/snippets/idempotency/workingWithLocalDynamoDB.test.ts @@ -0,0 +1,44 @@ +import { makeIdempotent } from '@aws-lambda-powertools/idempotency'; +import { DynamoDBPersistenceLayer } from '@aws-lambda-powertools/idempotency/dynamodb'; +import { DynamoDBClient } from '@aws-sdk/client-dynamodb'; +import type { Context } from 'aws-lambda'; +import { handler } from './workingWithLocalDynamoDB'; + +describe('Idempotent handler', () => { + const lambdaContext = { + functionName: 'foo-bar-function', + memoryLimitInMB: '128', + invokedFunctionArn: + 'arn:aws:lambda:eu-west-1:123456789012:function:foo-bar-function', + awsRequestId: 'c6af9ac6-7b61-11e6-9a41-93e812345678', + getRemainingTimeInMillis: () => 1234, + } as Context; + const ddbLocalClient = new DynamoDBClient({ + endpoint: 'http://localhost:8000', // 8000 for local DynamoDB and 4566 for LocalStack + }); + + const mockPersistenceStore = new DynamoDBPersistenceLayer({ + tableName: 'IdempotencyTable', + awsSdkV3Client: ddbLocalClient, + }); + + const idempotentHandler = makeIdempotent(handler, { + persistenceStore: mockPersistenceStore, + }); + + it('should return the same response', async () => { + const response = await idempotentHandler( + { + foo: 'bar', + }, + lambdaContext + ); + expect(response).toEqual({ + statusCode: 200, + body: JSON.stringify({ + paymentId: '123', + message: 'Payment created', + }), + }); + }); +}); diff --git a/examples/snippets/idempotency/workingWithLocalDynamoDB.ts b/examples/snippets/idempotency/workingWithLocalDynamoDB.ts new file mode 100644 index 0000000000..f48430456e --- /dev/null +++ b/examples/snippets/idempotency/workingWithLocalDynamoDB.ts @@ -0,0 +1,23 @@ +import { makeIdempotent } from '@aws-lambda-powertools/idempotency'; +import { DynamoDBPersistenceLayer } from '@aws-lambda-powertools/idempotency/dynamodb'; +import type { Context } from 'aws-lambda'; + +const ddbPersistenceStore = new DynamoDBPersistenceLayer({ + tableName: 'IdempotencyTable', +}); + +const handler = async (event: unknown, context: Context) => { + return { + statusCode: 200, + body: JSON.stringify({ + paymentId: '123', + message: 'Payment created', + }), + }; +}; + +const idempotentHandler = makeIdempotent(handler, { + persistenceStore: ddbPersistenceStore, +}); + +export { idempotentHandler, handler }; From ea2fc802ef343418a325ff012059bb32a13a4e92 Mon Sep 17 00:00:00 2001 From: Alexander Schueren Date: Tue, 13 Aug 2024 14:45:43 +0200 Subject: [PATCH 08/10] fix indent in readme examples --- packages/idempotency/README.md | 142 ++++++++++++++++----------------- 1 file changed, 71 insertions(+), 71 deletions(-) diff --git a/packages/idempotency/README.md b/packages/idempotency/README.md index 3212cdd1f7..6a3abc1dab 100644 --- a/packages/idempotency/README.md +++ b/packages/idempotency/README.md @@ -44,18 +44,18 @@ import type {Context, APIGatewayProxyEvent} from 'aws-lambda'; `` const persistenceStore = new DynamoDBPersistenceLayer({ - tableName: 'idempotencyTableName', + tableName: 'idempotencyTableName', }); const myHandler = async ( - event: APIGatewayProxyEvent, - _context: Context + event: APIGatewayProxyEvent, + _context: Context ): Promise => { - // your code goes here here + // your code goes here here }; export const handler = makeIdempotent(myHandler, { - persistenceStore, + persistenceStore, }); ``` @@ -67,24 +67,24 @@ import {DynamoDBPersistenceLayer} from '@aws-lambda-powertools/idempotency/dynam import type {Context, SQSEvent, SQSRecord} from 'aws-lambda'; const persistenceStore = new DynamoDBPersistenceLayer({ - tableName: 'idempotencyTableName', + tableName: 'idempotencyTableName', }); const processingFunction = async (payload: SQSRecord): Promise => { - // your code goes here here + // your code goes here here }; const processIdempotently = makeIdempotent(processingFunction, { - persistenceStore, + persistenceStore, }); export const handler = async ( - event: SQSEvent, - _context: Context + event: SQSEvent, + _context: Context ): Promise => { - for (const record of event.Records) { - await processIdempotently(record); - } + for (const record of event.Records) { + await processIdempotently(record); + } }; ``` @@ -97,26 +97,26 @@ import {DynamoDBPersistenceLayer} from '@aws-lambda-powertools/idempotency/dynam import type {Context, SQSEvent, SQSRecord} from 'aws-lambda'; const persistenceStore = new DynamoDBPersistenceLayer({ - tableName: 'idempotencyTableName', + tableName: 'idempotencyTableName', }); const processingFunction = async (payload: SQSRecord, customerId: string): Promise => { - // your code goes here here + // your code goes here here }; const processIdempotently = makeIdempotent(processingFunction, { - persistenceStore, - // this tells the utility to use the second argument (`customerId`) as the idempotency key - dataIndexArgument: 1, + persistenceStore, + // this tells the utility to use the second argument (`customerId`) as the idempotency key + dataIndexArgument: 1, }); export const handler = async ( - event: SQSEvent, - _context: Context + event: SQSEvent, + _context: Context ): Promise => { - for (const record of event.Records) { - await processIdempotently(record, 'customer-123'); - } + for (const record of event.Records) { + await processIdempotently(record, 'customer-123'); + } }; ``` @@ -133,17 +133,17 @@ const persistenceStore = new DynamoDBPersistenceLayer({ }); const myHandler = async ( - event: APIGatewayProxyEvent, - _context: Context + event: APIGatewayProxyEvent, + _context: Context ): Promise => { - // your code goes here here + // your code goes here here }; export const handler = makeIdempotent(myHandler, { - persistenceStore, - config: new IdempotencyConfig({ - eventKeyJmespath: 'requestContext.identity.user', - }), + persistenceStore, + config: new IdempotencyConfig({ + eventKeyJmespath: 'requestContext.identity.user', + }), }); ``` @@ -157,21 +157,21 @@ import {DynamoDBPersistenceLayer} from '@aws-lambda-powertools/idempotency/dynam import type {Context, APIGatewayProxyEvent} from 'aws-lambda'; const persistenceStore = new DynamoDBPersistenceLayer({ - tableName: 'idempotencyTableName', + tableName: 'idempotencyTableName', }); const myHandler = async ( - event: APIGatewayProxyEvent, - _context: Context + event: APIGatewayProxyEvent, + _context: Context ): Promise => { - // your code goes here here + // your code goes here here }; export const handler = makeIdempotent(myHandler, { - persistenceStore, - config: new IdempotencyConfig({ - eventKeyJmespath: 'powertools_json(body).["user", "productId"]', - }), + persistenceStore, + config: new IdempotencyConfig({ + eventKeyJmespath: 'powertools_json(body).["user", "productId"]', + }), }); ``` @@ -188,17 +188,17 @@ import {DynamoDBPersistenceLayer} from '@aws-lambda-powertools/idempotency/dynam import type {Context, APIGatewayProxyEvent} from 'aws-lambda'; const persistenceStore = new DynamoDBPersistenceLayer({ - tableName: 'idempotencyTableName', + tableName: 'idempotencyTableName', }); class MyHandler extends LambdaInterface { - @idempotent({persistenceStore: dynamoDBPersistenceLayer}) - public async handler( - event: APIGatewayProxyEvent, - context: Context - ): Promise { - // your code goes here here - } + @idempotent({persistenceStore: dynamoDBPersistenceLayer}) + public async handler( + event: APIGatewayProxyEvent, + context: Context + ): Promise { + // your code goes here here + } } const handlerClass = new MyHandler(); @@ -214,24 +214,24 @@ import {DynamoDBPersistenceLayer} from '@aws-lambda-powertools/idempotency/dynam import type {Context} from 'aws-lambda'; const persistenceStore = new DynamoDBPersistenceLayer({ - tableName: 'idempotencyTableName', + tableName: 'idempotencyTableName', }); class MyHandler extends LambdaInterface { - public async handler( - event: unknown, - context: Context - ): Promise { - for (const record of event.Records) { - await this.processIdempotently(record); - } + public async handler( + event: unknown, + context: Context + ): Promise { + for (const record of event.Records) { + await this.processIdempotently(record); } + } - @idempotent({persistenceStore: dynamoDBPersistenceLayer}) - private async process(record: unknown): Promise { - // process each code idempotently - } + @idempotent({persistenceStore: dynamoDBPersistenceLayer}) + private async process(record: unknown): Promise { + // process each code idempotently + } } const handlerClass = new MyHandler(); @@ -260,25 +260,25 @@ import middy from '@middy/core'; import type {Context, APIGatewayProxyEvent} from 'aws-lambda'; const persistenceStore = new DynamoDBPersistenceLayer({ - tableName: 'idempotencyTableName', + tableName: 'idempotencyTableName', }); const config = new IdempotencyConfig({ - hashFunction: 'md5', - useLocalCache: false, - expiresAfterSeconds: 3600, - throwOnNoIdempotencyKey: false, - eventKeyJmesPath: 'headers.idempotency-key', + hashFunction: 'md5', + useLocalCache: false, + expiresAfterSeconds: 3600, + throwOnNoIdempotencyKey: false, + eventKeyJmesPath: 'headers.idempotency-key', }); export const handler = middy( - async (_event: APIGatewayProxyEvent, _context: Context): Promise => { - // your code goes here here - } + async (_event: APIGatewayProxyEvent, _context: Context): Promise => { + // your code goes here here + } ).use( - makeHandlerIdempotent({ - config, - persistenceStore, - }) + makeHandlerIdempotent({ + config, + persistenceStore, + }) ); ``` From 3dcd3efdfc610d0b80cdbbbd6e37198d6b64a5ac Mon Sep 17 00:00:00 2001 From: Alexander Schueren Date: Tue, 13 Aug 2024 15:10:31 +0200 Subject: [PATCH 09/10] pass clientconfig instead of the client --- .../snippets/idempotency/workingWithLocalDynamoDB.test.ts | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/examples/snippets/idempotency/workingWithLocalDynamoDB.test.ts b/examples/snippets/idempotency/workingWithLocalDynamoDB.test.ts index adf4f18847..f3cfdddbc9 100644 --- a/examples/snippets/idempotency/workingWithLocalDynamoDB.test.ts +++ b/examples/snippets/idempotency/workingWithLocalDynamoDB.test.ts @@ -1,6 +1,5 @@ import { makeIdempotent } from '@aws-lambda-powertools/idempotency'; import { DynamoDBPersistenceLayer } from '@aws-lambda-powertools/idempotency/dynamodb'; -import { DynamoDBClient } from '@aws-sdk/client-dynamodb'; import type { Context } from 'aws-lambda'; import { handler } from './workingWithLocalDynamoDB'; @@ -13,13 +12,10 @@ describe('Idempotent handler', () => { awsRequestId: 'c6af9ac6-7b61-11e6-9a41-93e812345678', getRemainingTimeInMillis: () => 1234, } as Context; - const ddbLocalClient = new DynamoDBClient({ - endpoint: 'http://localhost:8000', // 8000 for local DynamoDB and 4566 for LocalStack - }); const mockPersistenceStore = new DynamoDBPersistenceLayer({ tableName: 'IdempotencyTable', - awsSdkV3Client: ddbLocalClient, + clientConfig: { endpoint: 'http://localhost:8000' }, // 8000 for local DynamoDB and 4566 for LocalStack }); const idempotentHandler = makeIdempotent(handler, { From aafdbb53f193b4ce986097311aa9029f9897117a Mon Sep 17 00:00:00 2001 From: Alexander Schueren Date: Tue, 13 Aug 2024 15:14:03 +0200 Subject: [PATCH 10/10] fix warnings in typedoc --- .../idempotency/src/persistence/BasePersistenceLayer.ts | 6 ++---- .../src/persistence/DynamoDBPersistenceLayer.ts | 1 - packages/idempotency/src/persistence/IdempotencyRecord.ts | 2 +- packages/idempotency/src/types/DynamoDBPersistence.ts | 2 -- packages/idempotency/src/types/index.ts | 7 +++++++ packages/idempotency/typedoc.json | 8 +++----- 6 files changed, 13 insertions(+), 13 deletions(-) diff --git a/packages/idempotency/src/persistence/BasePersistenceLayer.ts b/packages/idempotency/src/persistence/BasePersistenceLayer.ts index cd883c4218..51e48f8b4f 100644 --- a/packages/idempotency/src/persistence/BasePersistenceLayer.ts +++ b/packages/idempotency/src/persistence/BasePersistenceLayer.ts @@ -21,9 +21,7 @@ import { IdempotencyRecord } from './IdempotencyRecord.js'; * Base class for all persistence layers. This class provides the basic functionality for * saving, retrieving, and deleting idempotency records. It also provides the ability to * configure the persistence layer from the idempotency config. - * @abstract * @class - * @implements {BasePersistenceLayerInterface} */ abstract class BasePersistenceLayer implements BasePersistenceLayerInterface { public idempotencyKeyPrefix: string; @@ -143,8 +141,8 @@ abstract class BasePersistenceLayer implements BasePersistenceLayerInterface { * * The record is also saved to the local cache if local caching is enabled. * - * @param record - the stored record to validate against - * @param data - the data payload being processed and to be validated against the stored record + * @param storedDataRecord - the stored record to validate against + * @param processedData - the data payload being processed and to be validated against the stored record */ public processExistingRecord( storedDataRecord: IdempotencyRecord, diff --git a/packages/idempotency/src/persistence/DynamoDBPersistenceLayer.ts b/packages/idempotency/src/persistence/DynamoDBPersistenceLayer.ts index 37b75016cc..3c84f1850f 100644 --- a/packages/idempotency/src/persistence/DynamoDBPersistenceLayer.ts +++ b/packages/idempotency/src/persistence/DynamoDBPersistenceLayer.ts @@ -46,7 +46,6 @@ import { IdempotencyRecord } from './IdempotencyRecord.js'; * * @see https://docs.aws.amazon.com/AWSJavaScriptSDK/v3/latest/clients/client-dynamodb/index.html * @category Persistence Layer - * @implements {BasePersistenceLayer} */ class DynamoDBPersistenceLayer extends BasePersistenceLayer { private client: DynamoDBClient; diff --git a/packages/idempotency/src/persistence/IdempotencyRecord.ts b/packages/idempotency/src/persistence/IdempotencyRecord.ts index c5057e846f..2b99cadc4f 100644 --- a/packages/idempotency/src/persistence/IdempotencyRecord.ts +++ b/packages/idempotency/src/persistence/IdempotencyRecord.ts @@ -34,7 +34,7 @@ class IdempotencyRecord { /** * The idempotency record status can be COMPLETED, IN_PROGRESS or EXPIRED. * We check the status during idempotency processing to make sure we don't process an expired record and handle concurrent requests. - * @link {IdempotencyRecordStatusValue} + * {@link constants.IdempotencyRecordStatusValue | IdempotencyRecordStatusValue} * @private */ private status: IdempotencyRecordStatusValue; diff --git a/packages/idempotency/src/types/DynamoDBPersistence.ts b/packages/idempotency/src/types/DynamoDBPersistence.ts index 3a30a73639..97733e8732 100644 --- a/packages/idempotency/src/types/DynamoDBPersistence.ts +++ b/packages/idempotency/src/types/DynamoDBPersistence.ts @@ -33,7 +33,6 @@ interface DynamoDBPersistenceOptionsBase { * Interface for DynamoDBPersistenceOptions with clientConfig property. * * @interface - * @extends DynamoDBPersistenceOptionsBase * @property {DynamoDBClientConfig} [clientConfig] - Optional configuration to pass during client initialization, e.g. AWS region. * @property {never} [awsSdkV3Client] - This property should never be passed. */ @@ -50,7 +49,6 @@ interface DynamoDBPersistenceOptionsWithClientConfig * Interface for DynamoDBPersistenceOptions with awsSdkV3Client property. * * @interface - * @extends DynamoDBPersistenceOptionsBase * @property {DynamoDBClient} [awsSdkV3Client] - Optional AWS SDK v3 client to pass during DynamoDB client instantiation * @property {never} [clientConfig] - This property should never be passed. */ diff --git a/packages/idempotency/src/types/index.ts b/packages/idempotency/src/types/index.ts index 96dfca108d..c390a7d185 100644 --- a/packages/idempotency/src/types/index.ts +++ b/packages/idempotency/src/types/index.ts @@ -11,4 +11,11 @@ export type { IdempotencyLambdaHandlerOptions, IdempotencyHandlerOptions, ItempotentFunctionOptions, + AnyFunction, } from './IdempotencyOptions.js'; +export type { + DynamoDBPersistenceOptions, + DynamoDBPersistenceOptionsBase, + DynamoDBPersistenceOptionsWithClientConfig, + DynamoDBPersistenceOptionsWithClientInstance, +} from './DynamoDBPersistence.js'; diff --git a/packages/idempotency/typedoc.json b/packages/idempotency/typedoc.json index d1bb1f2c03..938073260b 100644 --- a/packages/idempotency/typedoc.json +++ b/packages/idempotency/typedoc.json @@ -1,13 +1,11 @@ { - "extends": [ - "../../typedoc.base.json" - ], + "extends": ["../../typedoc.base.json"], "entryPoints": [ "./src/index.ts", "./src/types/index.ts", - "./src/middleware/index.ts", + "./src/middleware/makeHandlerIdempotent.ts", "./src/persistence/index.ts", "./src/persistence/DynamoDBPersistenceLayer.ts" ], "readme": "README.md" -} \ No newline at end of file +}