From bcc17c28d401d386e93864a0eb008a455fca5a9c Mon Sep 17 00:00:00 2001 From: Alexander Melnyk Date: Tue, 26 Sep 2023 22:49:32 +0200 Subject: [PATCH 1/4] add decorator and tests --- .../idempotency/src/idempotencyDecorator.ts | 164 ++++++++++ packages/idempotency/src/index.ts | 1 + .../tests/unit/idempotencyDecorator.test.ts | 304 ++++++++++++++++++ 3 files changed, 469 insertions(+) create mode 100644 packages/idempotency/src/idempotencyDecorator.ts create mode 100644 packages/idempotency/tests/unit/idempotencyDecorator.test.ts diff --git a/packages/idempotency/src/idempotencyDecorator.ts b/packages/idempotency/src/idempotencyDecorator.ts new file mode 100644 index 0000000000..d3f1550779 --- /dev/null +++ b/packages/idempotency/src/idempotencyDecorator.ts @@ -0,0 +1,164 @@ +import type { Context, Handler } from 'aws-lambda'; +import { + AnyFunction, + IdempotencyLambdaHandlerOptions, + ItempotentFunctionOptions, +} from './types'; +import { IdempotencyHandler } from './IdempotencyHandler'; +import { IdempotencyConfig } from './IdempotencyConfig'; +import { makeIdempotent } from './makeIdempotent'; + +const isContext = (arg: unknown): arg is Context => { + return ( + arg !== undefined && + arg !== null && + typeof arg === 'object' && + 'getRemainingTimeInMillis' in arg + ); +}; + +const isFnHandler = ( + fn: AnyFunction, + args: Parameters +): fn is Handler => { + // get arguments of function + return ( + fn !== undefined && + fn !== null && + typeof fn === 'function' && + isContext(args[1]) + ); +}; + +const isOptionsWithDataIndexArgument = ( + options: unknown +): options is IdempotencyLambdaHandlerOptions & { + dataIndexArgument: number; +} => { + return ( + options !== undefined && + options !== null && + typeof options === 'object' && + 'dataIndexArgument' in options + ); +}; + +const idempotent = function ( + options: ItempotentFunctionOptions> +): ( + target: unknown, + propertyKey: string, + descriptor: PropertyDescriptor +) => PropertyDescriptor { + return function ( + _target: unknown, + _propertyKey: string, + descriptor: PropertyDescriptor + ) { + const childFunction = descriptor.value; + descriptor.value = makeIdempotent(childFunction, options); + descriptor.value = function (...args: Parameters) { + const { persistenceStore, config } = options; + const idempotencyConfig = config ? config : new IdempotencyConfig({}); + + if (!idempotencyConfig.isEnabled()) + return childFunction.apply(this, args); + + let functionPayloadtoBeHashed; + + if (isFnHandler(childFunction, args)) { + idempotencyConfig.registerLambdaContext(args[1]); + functionPayloadtoBeHashed = args[0]; + } else { + if (isOptionsWithDataIndexArgument(options)) { + functionPayloadtoBeHashed = args[options.dataIndexArgument]; + } else { + functionPayloadtoBeHashed = args[0]; + } + } + + return new IdempotencyHandler({ + functionToMakeIdempotent: childFunction, + idempotencyConfig: idempotencyConfig, + persistenceStore: persistenceStore, + functionArguments: args, + functionPayloadToBeHashed: functionPayloadtoBeHashed, + }).handle(); + }; + + return descriptor; + }; +}; + +/** + * Use this decorator to make your lambda handler itempotent. + * You need to provide a peristance layer to store the idempotency information. + * At the moment we only support `DynamodbPersistenceLayer`. + * > **Note**: + * > decorators are an exeperimental feature in typescript and may change in the future. + * > To enable decoratopr support in your project, you need to enable the `experimentalDecorators` compiler option in your tsconfig.json file. + * @example + * ```ts + * import { + * DynamoDBPersistenceLayer, + * idempotentLambdaHandler + * } from '@aws-lambda-powertools/idempotency' + * + * class MyLambdaFunction { + * @idempotentLambdaHandler({ persistenceStore: new DynamoDBPersistenceLayer() }) + * async handler(event: any, context: any) { + * return "Hello World"; + * } + * } + * export myLambdaHandler new MyLambdaFunction(); + * export const handler = myLambdaHandler.handler.bind(myLambdaHandler); + * ``` + * @see {@link DynamoDBPersistenceLayer} + * @see https://www.typescriptlang.org/docs/handbook/decorators.html + */ +// const idempotentLambdaHandler = function ( +// options: IdempotencyLambdaHandlerOptions +// ): ( +// target: unknown, +// propertyKey: string, +// descriptor: PropertyDescriptor +// ) => PropertyDescriptor { +// return idempotent(options); +// }; +/** + * Use this decorator to make any class function idempotent. + * Similar to the `idempotentLambdaHandler` decorator, you need to provide a persistence layer to store the idempotency information. + * @example + * ```ts + * import { + * DynamoDBPersistenceLayer, + * idempotentFunction + * } from '@aws-lambda-powertools/idempotency' + * + * class MyClass { + * + * public async handler(_event: any, _context: any) { + * for(const record of _event.records){ + * await this.process(record); + * } + * } + * + * @idempotentFunction({ persistenceStore: new DynamoDBPersistenceLayer() }) + * public async process(record: Record> +// ): ( +// target: unknown, +// propertyKey: string, +// descriptor: PropertyDescriptor +// ) => PropertyDescriptor { +// return idempotent(options); +// }; + +export { idempotent }; diff --git a/packages/idempotency/src/index.ts b/packages/idempotency/src/index.ts index c8a20f002f..0b86f82a69 100644 --- a/packages/idempotency/src/index.ts +++ b/packages/idempotency/src/index.ts @@ -1,4 +1,5 @@ export * from './errors'; export * from './IdempotencyConfig'; export * from './makeIdempotent'; +export * from './idempotencyDecorator'; export { IdempotencyRecordStatus } from './constants'; diff --git a/packages/idempotency/tests/unit/idempotencyDecorator.test.ts b/packages/idempotency/tests/unit/idempotencyDecorator.test.ts new file mode 100644 index 0000000000..1a293d8569 --- /dev/null +++ b/packages/idempotency/tests/unit/idempotencyDecorator.test.ts @@ -0,0 +1,304 @@ +/** + * Test Function Wrapper + * + * @group unit/idempotency/decorator + */ + +import { BasePersistenceLayer, IdempotencyRecord } from '../../src/persistence'; +import { idempotent } from '../../src/'; +import type { IdempotencyRecordOptions } from '../../src/types'; +import { + IdempotencyAlreadyInProgressError, + IdempotencyInconsistentStateError, + IdempotencyItemAlreadyExistsError, + IdempotencyPersistenceLayerError, +} from '../../src/errors'; +import { IdempotencyConfig } from '../../src'; +import { Context } from 'aws-lambda'; +import { helloworldContext } from '@aws-lambda-powertools/commons/lib/samples/resources/contexts'; +import { IdempotencyRecordStatus } from '../../src/constants'; + +const mockSaveInProgress = jest + .spyOn(BasePersistenceLayer.prototype, 'saveInProgress') + .mockImplementation(); +const mockSaveSuccess = jest + .spyOn(BasePersistenceLayer.prototype, 'saveSuccess') + .mockImplementation(); +const mockGetRecord = jest + .spyOn(BasePersistenceLayer.prototype, 'getRecord') + .mockImplementation(); + +const dummyContext = helloworldContext; + +const mockConfig: IdempotencyConfig = new IdempotencyConfig({}); + +class PersistenceLayerTestClass extends BasePersistenceLayer { + protected _deleteRecord = jest.fn(); + protected _getRecord = jest.fn(); + protected _putRecord = jest.fn(); + protected _updateRecord = jest.fn(); +} + +const functionalityToDecorate = jest.fn(); + +class TestinClassWithLambdaHandler { + @idempotent({ + persistenceStore: new PersistenceLayerTestClass(), + }) + public testing(record: Record, _context: Context): string { + functionalityToDecorate(record); + + return 'Hi'; + } +} + +class TestingClassWithFunctionDecorator { + public handler(record: Record, context: Context): string { + mockConfig.registerLambdaContext(context); + + return this.proccessRecord(record, 'bar'); + } + + @idempotent({ + persistenceStore: new PersistenceLayerTestClass(), + config: mockConfig, + dataIndexArgument: 0, + }) + public proccessRecord(record: Record, _foo: string): string { + functionalityToDecorate(record); + + return 'Processed Record'; + } +} + +describe('Given a class with a function to decorate', (classWithLambdaHandler = new TestinClassWithLambdaHandler(), classWithFunctionDecorator = new TestingClassWithFunctionDecorator()) => { + const keyValueToBeSaved = 'thisWillBeSaved'; + const inputRecord = { + testingKey: keyValueToBeSaved, + otherKey: 'thisWillNot', + }; + beforeEach(() => jest.clearAllMocks()); + + describe('When wrapping a function with no previous executions', () => { + beforeEach(async () => { + await classWithFunctionDecorator.handler(inputRecord, dummyContext); + }); + + test('Then it will save the record to INPROGRESS', () => { + expect(mockSaveInProgress).toBeCalledWith( + inputRecord, + dummyContext.getRemainingTimeInMillis() + ); + }); + + test('Then it will call the function that was decorated', () => { + expect(functionalityToDecorate).toBeCalledWith(inputRecord); + }); + + test('Then it will save the record to COMPLETED with function return value', () => { + expect(mockSaveSuccess).toBeCalledWith(inputRecord, 'Processed Record'); + }); + }); + describe('When wrapping a handler function with no previous executions', () => { + beforeEach(async () => { + await classWithLambdaHandler.testing(inputRecord, dummyContext); + }); + + test('Then it will save the record to INPROGRESS', () => { + expect(mockSaveInProgress).toBeCalledWith( + inputRecord, + dummyContext.getRemainingTimeInMillis() + ); + }); + + test('Then it will call the function that was decorated', () => { + expect(functionalityToDecorate).toBeCalledWith(inputRecord); + }); + + test('Then it will save the record to COMPLETED with function return value', () => { + expect(mockSaveSuccess).toBeCalledWith(inputRecord, 'Hi'); + }); + }); + + describe('When decorating a function with previous execution that is INPROGRESS', () => { + let resultingError: Error; + beforeEach(async () => { + mockSaveInProgress.mockRejectedValue( + new IdempotencyItemAlreadyExistsError() + ); + const idempotencyOptions: IdempotencyRecordOptions = { + idempotencyKey: 'key', + status: IdempotencyRecordStatus.INPROGRESS, + }; + mockGetRecord.mockResolvedValue( + new IdempotencyRecord(idempotencyOptions) + ); + try { + await classWithLambdaHandler.testing(inputRecord, dummyContext); + } catch (e) { + resultingError = e as Error; + } + }); + + test('Then it will attempt to save the record to INPROGRESS', () => { + expect(mockSaveInProgress).toBeCalledWith( + inputRecord, + dummyContext.getRemainingTimeInMillis() + ); + }); + + test('Then it will get the previous execution record', () => { + expect(mockGetRecord).toBeCalledWith(inputRecord); + }); + + test('Then it will not call the function that was decorated', () => { + expect(functionalityToDecorate).not.toBeCalled(); + }); + + test('Then an IdempotencyAlreadyInProgressError is thrown', () => { + expect(resultingError).toBeInstanceOf(IdempotencyAlreadyInProgressError); + }); + }); + + describe('When decorating a function with previous execution that is EXPIRED', () => { + let resultingError: Error; + beforeEach(async () => { + mockSaveInProgress.mockRejectedValue( + new IdempotencyItemAlreadyExistsError() + ); + const idempotencyOptions: IdempotencyRecordOptions = { + idempotencyKey: 'key', + status: IdempotencyRecordStatus.EXPIRED, + }; + mockGetRecord.mockResolvedValue( + new IdempotencyRecord(idempotencyOptions) + ); + try { + await classWithLambdaHandler.testing(inputRecord, dummyContext); + } catch (e) { + resultingError = e as Error; + } + }); + + test('Then it will attempt to save the record to INPROGRESS', () => { + expect(mockSaveInProgress).toBeCalledWith( + inputRecord, + dummyContext.getRemainingTimeInMillis() + ); + }); + + test('Then it will get the previous execution record', () => { + expect(mockGetRecord).toBeCalledWith(inputRecord); + }); + + test('Then it will not call the function that was decorated', () => { + expect(functionalityToDecorate).not.toBeCalled(); + }); + + test('Then an IdempotencyInconsistentStateError is thrown', () => { + expect(resultingError).toBeInstanceOf(IdempotencyInconsistentStateError); + }); + }); + + describe('When wrapping a function with previous execution that is COMPLETED', () => { + beforeEach(async () => { + mockSaveInProgress.mockRejectedValue( + new IdempotencyItemAlreadyExistsError() + ); + const idempotencyOptions: IdempotencyRecordOptions = { + idempotencyKey: 'key', + status: IdempotencyRecordStatus.COMPLETED, + responseData: 'Hi', + }; + + mockGetRecord.mockResolvedValue( + new IdempotencyRecord(idempotencyOptions) + ); + await classWithLambdaHandler.testing(inputRecord, dummyContext); + }); + + test('Then it will attempt to save the record to INPROGRESS', () => { + expect(mockSaveInProgress).toBeCalledWith( + inputRecord, + dummyContext.getRemainingTimeInMillis() + ); + }); + + test('Then it will get the previous execution record', () => { + expect(mockGetRecord).toBeCalledWith(inputRecord); + }); + + test('Then it will not call decorated functionality', () => { + expect(functionalityToDecorate).not.toBeCalledWith(inputRecord); + }); + }); + + describe('When wrapping a function with issues saving the record', () => { + class TestinClassWithLambdaHandlerWithConfig { + @idempotent({ + persistenceStore: new PersistenceLayerTestClass(), + config: new IdempotencyConfig({ lambdaContext: dummyContext }), + }) + public testing(record: Record): string { + functionalityToDecorate(record); + + return 'Hi'; + } + } + + let resultingError: Error; + beforeEach(async () => { + mockSaveInProgress.mockRejectedValue(new Error('RandomError')); + const classWithLambdaHandlerWithConfig = + new TestinClassWithLambdaHandlerWithConfig(); + try { + await classWithLambdaHandlerWithConfig.testing(inputRecord); + } catch (e) { + resultingError = e as Error; + } + }); + + test('Then it will attempt to save the record to INPROGRESS', () => { + expect(mockSaveInProgress).toBeCalledWith( + inputRecord, + dummyContext.getRemainingTimeInMillis() + ); + }); + + test('Then an IdempotencyPersistenceLayerError is thrown', () => { + expect(resultingError).toBeInstanceOf(IdempotencyPersistenceLayerError); + }); + }); + + describe('When idempotency is disabled', () => { + beforeAll(async () => { + process.env.POWERTOOLS_IDEMPOTENCY_DISABLED = 'true'; + class TestingClassWithIdempotencyDisabled { + @idempotent({ + persistenceStore: new PersistenceLayerTestClass(), + config: new IdempotencyConfig({ lambdaContext: dummyContext }), + }) + public testing( + record: Record, + _context: Context + ): string { + functionalityToDecorate(record); + + return 'Hi'; + } + } + const classWithoutIdempotencyDisabled = + new TestingClassWithIdempotencyDisabled(); + await classWithoutIdempotencyDisabled.testing(inputRecord, dummyContext); + }); + + test('Then it will skip ipdemotency', async () => { + expect(mockSaveInProgress).not.toBeCalled(); + expect(mockSaveSuccess).not.toBeCalled(); + }); + + afterAll(() => { + delete process.env.POWERTOOLS_IDEMPOTENCY_DISABLED; + }); + }); +}); From b14afea77d0c03100d5a41279ee41d723ed74bef Mon Sep 17 00:00:00 2001 From: Alexander Melnyk Date: Tue, 26 Sep 2023 22:59:25 +0200 Subject: [PATCH 2/4] remove unused code, merged comments --- .../idempotency/src/idempotencyDecorator.ts | 149 ++++-------------- 1 file changed, 27 insertions(+), 122 deletions(-) diff --git a/packages/idempotency/src/idempotencyDecorator.ts b/packages/idempotency/src/idempotencyDecorator.ts index d3f1550779..53e17f3ef3 100644 --- a/packages/idempotency/src/idempotencyDecorator.ts +++ b/packages/idempotency/src/idempotencyDecorator.ts @@ -1,102 +1,11 @@ -import type { Context, Handler } from 'aws-lambda'; -import { - AnyFunction, - IdempotencyLambdaHandlerOptions, - ItempotentFunctionOptions, -} from './types'; -import { IdempotencyHandler } from './IdempotencyHandler'; -import { IdempotencyConfig } from './IdempotencyConfig'; +import { AnyFunction, ItempotentFunctionOptions } from './types'; import { makeIdempotent } from './makeIdempotent'; -const isContext = (arg: unknown): arg is Context => { - return ( - arg !== undefined && - arg !== null && - typeof arg === 'object' && - 'getRemainingTimeInMillis' in arg - ); -}; - -const isFnHandler = ( - fn: AnyFunction, - args: Parameters -): fn is Handler => { - // get arguments of function - return ( - fn !== undefined && - fn !== null && - typeof fn === 'function' && - isContext(args[1]) - ); -}; - -const isOptionsWithDataIndexArgument = ( - options: unknown -): options is IdempotencyLambdaHandlerOptions & { - dataIndexArgument: number; -} => { - return ( - options !== undefined && - options !== null && - typeof options === 'object' && - 'dataIndexArgument' in options - ); -}; - -const idempotent = function ( - options: ItempotentFunctionOptions> -): ( - target: unknown, - propertyKey: string, - descriptor: PropertyDescriptor -) => PropertyDescriptor { - return function ( - _target: unknown, - _propertyKey: string, - descriptor: PropertyDescriptor - ) { - const childFunction = descriptor.value; - descriptor.value = makeIdempotent(childFunction, options); - descriptor.value = function (...args: Parameters) { - const { persistenceStore, config } = options; - const idempotencyConfig = config ? config : new IdempotencyConfig({}); - - if (!idempotencyConfig.isEnabled()) - return childFunction.apply(this, args); - - let functionPayloadtoBeHashed; - - if (isFnHandler(childFunction, args)) { - idempotencyConfig.registerLambdaContext(args[1]); - functionPayloadtoBeHashed = args[0]; - } else { - if (isOptionsWithDataIndexArgument(options)) { - functionPayloadtoBeHashed = args[options.dataIndexArgument]; - } else { - functionPayloadtoBeHashed = args[0]; - } - } - - return new IdempotencyHandler({ - functionToMakeIdempotent: childFunction, - idempotencyConfig: idempotencyConfig, - persistenceStore: persistenceStore, - functionArguments: args, - functionPayloadToBeHashed: functionPayloadtoBeHashed, - }).handle(); - }; - - return descriptor; - }; -}; - /** * Use this decorator to make your lambda handler itempotent. * You need to provide a peristance layer to store the idempotency information. * At the moment we only support `DynamodbPersistenceLayer`. - * > **Note**: - * > decorators are an exeperimental feature in typescript and may change in the future. - * > To enable decoratopr support in your project, you need to enable the `experimentalDecorators` compiler option in your tsconfig.json file. + * * @example * ```ts * import { @@ -105,7 +14,7 @@ const idempotent = function ( * } from '@aws-lambda-powertools/idempotency' * * class MyLambdaFunction { - * @idempotentLambdaHandler({ persistenceStore: new DynamoDBPersistenceLayer() }) + * @idempotent({ persistenceStore: new DynamoDBPersistenceLayer() }) * async handler(event: any, context: any) { * return "Hello World"; * } @@ -113,21 +22,8 @@ const idempotent = function ( * export myLambdaHandler new MyLambdaFunction(); * export const handler = myLambdaHandler.handler.bind(myLambdaHandler); * ``` - * @see {@link DynamoDBPersistenceLayer} - * @see https://www.typescriptlang.org/docs/handbook/decorators.html - */ -// const idempotentLambdaHandler = function ( -// options: IdempotencyLambdaHandlerOptions -// ): ( -// target: unknown, -// propertyKey: string, -// descriptor: PropertyDescriptor -// ) => PropertyDescriptor { -// return idempotent(options); -// }; -/** - * Use this decorator to make any class function idempotent. - * Similar to the `idempotentLambdaHandler` decorator, you need to provide a persistence layer to store the idempotency information. + * + * Similar to decoratoring a handler you can use the decorator on any other function. * @example * ```ts * import { @@ -143,22 +39,31 @@ const idempotent = function ( * } * } * - * @idempotentFunction({ persistenceStore: new DynamoDBPersistenceLayer() }) + * @idemptent({ persistenceStore: new DynamoDBPersistenceLayer() }) * public async process(record: Record> -// ): ( -// target: unknown, -// propertyKey: string, -// descriptor: PropertyDescriptor -// ) => PropertyDescriptor { -// return idempotent(options); -// }; +const idempotent = function ( + options: ItempotentFunctionOptions> +): ( + target: unknown, + propertyKey: string, + descriptor: PropertyDescriptor +) => PropertyDescriptor { + return function ( + _target: unknown, + _propertyKey: string, + descriptor: PropertyDescriptor + ) { + const childFunction = descriptor.value; + descriptor.value = makeIdempotent(childFunction, options); + + return descriptor; + }; +}; export { idempotent }; From 6a99f18e183bd439a9b199273cf898ce8e5b911d Mon Sep 17 00:00:00 2001 From: Alexander Melnyk Date: Thu, 28 Sep 2023 16:04:32 +0200 Subject: [PATCH 3/4] add idempotecy decorator to docs --- .../idempotency/idempotentDecoratorBase.ts | 28 +++++++++++++++++++ docs/utilities/idempotency.md | 16 +++++++++++ .../idempotentDecorator.test.FunctionCode.ts | 0 .../tests/e2e/idempotentDecorator.test.ts | 0 4 files changed, 44 insertions(+) create mode 100644 docs/snippets/idempotency/idempotentDecoratorBase.ts create mode 100644 packages/idempotency/tests/e2e/idempotentDecorator.test.FunctionCode.ts create mode 100644 packages/idempotency/tests/e2e/idempotentDecorator.test.ts diff --git a/docs/snippets/idempotency/idempotentDecoratorBase.ts b/docs/snippets/idempotency/idempotentDecoratorBase.ts new file mode 100644 index 0000000000..a14c2b26e1 --- /dev/null +++ b/docs/snippets/idempotency/idempotentDecoratorBase.ts @@ -0,0 +1,28 @@ +import type { Context } from 'aws-lambda'; +import { LambdaInterface } from '@aws-lambda-powertools/commons'; +import { + IdempotencyConfig, + idempotent, +} from '@aws-lambda-powertools/idempotency'; +import { DynamoDBPersistenceLayer } from '@aws-lambda-powertools/idempotency/dynamodb'; +import { Request, Response } from './types'; + +const dynamoDBPersistenceLayer = new DynamoDBPersistenceLayer({ + tableName: 'idempotencyTableName', +}); + +const config = new IdempotencyConfig({}); + +class MyLambda implements LambdaInterface { + @idempotent({ persistenceStore: dynamoDBPersistenceLayer, config: config }) + public async handler(_event: Request, _context: Context): Promise { + // ... process your event + return { + message: 'success', + statusCode: 200, + }; + } +} + +const defaultLambda = new MyLambda(); +export const handler = defaultLambda.handler.bind(defaultLambda); diff --git a/docs/utilities/idempotency.md b/docs/utilities/idempotency.md index 3100fffa23..2a770ec6ba 100644 --- a/docs/utilities/idempotency.md +++ b/docs/utilities/idempotency.md @@ -160,6 +160,22 @@ When using `makeIdempotent` on arbitrary functions, you can tell us which argume The function this example has two arguments, note that while wrapping it with the `makeIdempotent` high-order function, we specify the `dataIndexArgument` as `1` to tell the decorator that the second argument is the one that contains the data we should use to make the function idempotent. Remember that arguments are zero-indexed, so the first argument is `0`, the second is `1`, and so on. +### Idempotent Decorator + +You can also use the `@idempotent` decorator to make your Lambda handler idempotent, similar to the `makeIdempotent` function wrapper. + +=== "index.ts" + + ```typescript hl_lines="17" + --8<-- "docs/snippets/idempotency/idempotentDecoratorBase.ts" + ``` + +=== "types.ts" + + ```typescript + +You can use the decorator on your Lambda handler or on any function that returns a response to make it idempotent. This is useful when you want to make a specific logic idempotent, for example when your Lambda handler performs multiple side effects and you only want to make a specific one idempotent. +The configuration options for the `@idempotent` decorator are the same as the ones for the `makeIdempotent` function wrapper. ### MakeHandlerIdempotent Middy middleware diff --git a/packages/idempotency/tests/e2e/idempotentDecorator.test.FunctionCode.ts b/packages/idempotency/tests/e2e/idempotentDecorator.test.FunctionCode.ts new file mode 100644 index 0000000000..e69de29bb2 diff --git a/packages/idempotency/tests/e2e/idempotentDecorator.test.ts b/packages/idempotency/tests/e2e/idempotentDecorator.test.ts new file mode 100644 index 0000000000..e69de29bb2 From ae0649167d39d7e5c3f870f3d66b3432dae7d918 Mon Sep 17 00:00:00 2001 From: Alexander Melnyk Date: Thu, 28 Sep 2023 16:09:53 +0200 Subject: [PATCH 4/4] add idempotency decorator tests --- packages/idempotency/README.md | 66 ++- .../idempotentDecorator.test.FunctionCode.ts | 164 ++++++ .../tests/e2e/idempotentDecorator.test.ts | 474 ++++++++++++++++++ 3 files changed, 703 insertions(+), 1 deletion(-) diff --git a/packages/idempotency/README.md b/packages/idempotency/README.md index 99e8f812e8..c295c5ef1d 100644 --- a/packages/idempotency/README.md +++ b/packages/idempotency/README.md @@ -9,6 +9,7 @@ You can use the package in both TypeScript and JavaScript code bases. - [Key features](#key-features) - [Usage](#usage) - [Function wrapper](#function-wrapper) + - [Decorator](#decorator) - [Middy middleware](#middy-middleware) - [DynamoDB persistence layer](#dynamodb-persistence-layer) - [Contribute](#contribute) @@ -24,7 +25,7 @@ You can use the package in both TypeScript and JavaScript code bases. ## Intro This package provides a utility to implement idempotency in your Lambda functions. -You can either use it to wrap a function, or as Middy middleware to make your AWS Lambda handler idempotent. +You can either use it to wrap a function, decorate a function, or as Middy middleware to make your AWS Lambda handler idempotent. The current implementation provides a persistence layer for Amazon DynamoDB, which offers a variety of configuration options. You can also bring your own persistence layer by extending the `BasePersistenceLayer` class. @@ -163,6 +164,69 @@ export const handler = makeIdempotent(myHandler, { Check the [docs](https://docs.powertools.aws.dev/lambda/typescript/latest/utilities/idempotency/) for more examples. +### Decorator + +You can make any function idempotent, and safe to retry, by decorating it using the `@idempotent` decorator. + +```ts +import { idempotent } from '@aws-lambda-powertools/idempotency'; +import { LambdaInterface } from '@aws-lambda-powertools/commons'; +import { DynamoDBPersistenceLayer } from '@aws-lambda-powertools/idempotency/dynamodb'; +import type { Context, APIGatewayProxyEvent } from 'aws-lambda'; + +const persistenceStore = new DynamoDBPersistenceLayer({ + tableName: 'idempotencyTableName', +}); + +class MyHandler extends LambdaInterface { + @idempotent({ persistenceStore: dynamoDBPersistenceLayer }) + public async handler( + event: APIGatewayProxyEvent, + context: Context + ): Promise { + // your code goes here here + } +} + +const handlerClass = new MyHandler(); +export const handler = handlerClass.handler.bind(handlerClass); +``` + +Using the same decorator, you can also make any other arbitrary function idempotent. + +```ts +import { idempotent } from '@aws-lambda-powertools/idempotency'; +import { LambdaInterface } from '@aws-lambda-powertools/commons'; +import { DynamoDBPersistenceLayer } from '@aws-lambda-powertools/idempotency/dynamodb'; +import type { Context } from 'aws-lambda'; + +const persistenceStore = new DynamoDBPersistenceLayer({ + tableName: 'idempotencyTableName', +}); + +class MyHandler extends LambdaInterface { + + public async handler( + event: unknown, + context: Context + ): Promise { + for(const record of event.Records) { + await this.processIdempotently(record); + } + } + + @idempotent({ persistenceStore: dynamoDBPersistenceLayer }) + private async process(record: unknown): Promise { + // process each code idempotently + } +} + +const handlerClass = new MyHandler(); +export const handler = handlerClass.handler.bind(handlerClass); +``` + +The decorator configuration options are identical with the ones of the `makeIdempotent` function. Check the [docs](https://docs.powertools.aws.dev/lambda/typescript/latest/utilities/idempotency/) for more examples. + ### Middy middleware If instead you use Middy, you can use the `makeHandlerIdempotent` middleware. When using the middleware your Lambda handler becomes idempotent. diff --git a/packages/idempotency/tests/e2e/idempotentDecorator.test.FunctionCode.ts b/packages/idempotency/tests/e2e/idempotentDecorator.test.FunctionCode.ts index e69de29bb2..8dd29c0b29 100644 --- a/packages/idempotency/tests/e2e/idempotentDecorator.test.FunctionCode.ts +++ b/packages/idempotency/tests/e2e/idempotentDecorator.test.FunctionCode.ts @@ -0,0 +1,164 @@ +import type { Context } from 'aws-lambda'; +import { LambdaInterface } from '@aws-lambda-powertools/commons'; +import { idempotent } from '../../src'; +import { Logger } from '../../../logger'; +import { DynamoDBPersistenceLayer } from '../../src/persistence/DynamoDBPersistenceLayer'; +import { IdempotencyConfig } from '../../src/'; + +const IDEMPOTENCY_TABLE_NAME = + process.env.IDEMPOTENCY_TABLE_NAME || 'table_name'; +const dynamoDBPersistenceLayer = new DynamoDBPersistenceLayer({ + tableName: IDEMPOTENCY_TABLE_NAME, +}); + +const dynamoDBPersistenceLayerCustomized = new DynamoDBPersistenceLayer({ + tableName: IDEMPOTENCY_TABLE_NAME, + dataAttr: 'dataAttr', + keyAttr: 'customId', + expiryAttr: 'expiryAttr', + statusAttr: 'statusAttr', + inProgressExpiryAttr: 'inProgressExpiryAttr', + staticPkValue: 'staticPkValue', + validationKeyAttr: 'validationKeyAttr', +}); + +const config = new IdempotencyConfig({}); + +class DefaultLambda implements LambdaInterface { + @idempotent({ persistenceStore: dynamoDBPersistenceLayer }) + public async handler( + _event: Record, + _context: Context + ): Promise { + logger.info(`Got test event: ${JSON.stringify(_event)}`); + // sleep to enforce error with parallel execution + await new Promise((resolve) => setTimeout(resolve, 1000)); + + return 'Hello World'; + } + + @idempotent({ + persistenceStore: dynamoDBPersistenceLayerCustomized, + config: config, + }) + public async handlerCustomized( + event: { foo: string }, + context: Context + ): Promise { + config.registerLambdaContext(context); + logger.info('Processed event', { details: event.foo }); + + return event.foo; + } + + @idempotent({ + persistenceStore: dynamoDBPersistenceLayer, + config: new IdempotencyConfig({ + useLocalCache: false, + expiresAfterSeconds: 1, + eventKeyJmesPath: 'foo', + }), + }) + public async handlerExpired( + event: { foo: string; invocation: number }, + context: Context + ): Promise<{ foo: string; invocation: number }> { + logger.addContext(context); + + logger.info('Processed event', { details: event.foo }); + + return { + foo: event.foo, + invocation: event.invocation, + }; + } + + @idempotent({ persistenceStore: dynamoDBPersistenceLayer }) + public async handlerParallel( + event: { foo: string }, + context: Context + ): Promise { + logger.addContext(context); + + await new Promise((resolve) => setTimeout(resolve, 1500)); + + logger.info('Processed event', { details: event.foo }); + + return event.foo; + } + + @idempotent({ + persistenceStore: dynamoDBPersistenceLayer, + config: new IdempotencyConfig({ + eventKeyJmesPath: 'foo', + }), + }) + public async handlerTimeout( + event: { foo: string; invocation: number }, + context: Context + ): Promise<{ foo: string; invocation: number }> { + logger.addContext(context); + + if (event.invocation === 0) { + await new Promise((resolve) => setTimeout(resolve, 4000)); + } + + logger.info('Processed event', { + details: event.foo, + }); + + return { + foo: event.foo, + invocation: event.invocation, + }; + } +} + +const defaultLambda = new DefaultLambda(); +const handler = defaultLambda.handler.bind(defaultLambda); +const handlerParallel = defaultLambda.handlerParallel.bind(defaultLambda); + +const handlerCustomized = defaultLambda.handlerCustomized.bind(defaultLambda); + +const handlerTimeout = defaultLambda.handlerTimeout.bind(defaultLambda); + +const handlerExpired = defaultLambda.handlerExpired.bind(defaultLambda); + +const logger = new Logger(); + +class LambdaWithKeywordArgument implements LambdaInterface { + public async handler( + event: { id: string }, + _context: Context + ): Promise { + config.registerLambdaContext(_context); + await this.process(event.id, 'bar'); + + return 'Hello World Keyword Argument'; + } + + @idempotent({ + persistenceStore: dynamoDBPersistenceLayer, + config: config, + dataIndexArgument: 1, + }) + public async process(id: string, foo: string): Promise { + logger.info('Got test event', { id, foo }); + + return 'idempotent result: ' + foo; + } +} + +const handlerDataIndexArgument = new LambdaWithKeywordArgument(); +const handlerWithKeywordArgument = handlerDataIndexArgument.handler.bind( + handlerDataIndexArgument +); + +export { + handler, + handlerCustomized, + handlerExpired, + handlerWithKeywordArgument, + handlerTimeout, + handlerParallel, +}; diff --git a/packages/idempotency/tests/e2e/idempotentDecorator.test.ts b/packages/idempotency/tests/e2e/idempotentDecorator.test.ts index e69de29bb2..87293d0eb3 100644 --- a/packages/idempotency/tests/e2e/idempotentDecorator.test.ts +++ b/packages/idempotency/tests/e2e/idempotentDecorator.test.ts @@ -0,0 +1,474 @@ +/** + * Test idempotency decorator + * + * @group e2e/idempotency + */ +import { DynamoDBClient } from '@aws-sdk/client-dynamodb'; +import { + RESOURCE_NAME_PREFIX, + SETUP_TIMEOUT, + TEARDOWN_TIMEOUT, + TEST_CASE_TIMEOUT, +} from './constants'; +import { ScanCommand } from '@aws-sdk/lib-dynamodb'; +import { createHash } from 'node:crypto'; +import { + invokeFunction, + isValidRuntimeKey, + TestInvocationLogs, + TestStack, +} from '@aws-lambda-powertools/testing-utils'; +import { IdempotencyTestNodejsFunctionAndDynamoTable } from '../helpers/resources'; +import { join } from 'node:path'; +import { Duration } from 'aws-cdk-lib'; +import { AttributeType } from 'aws-cdk-lib/aws-dynamodb'; + +const runtime: string = process.env.RUNTIME || 'nodejs18x'; + +if (!isValidRuntimeKey(runtime)) { + throw new Error(`Invalid runtime key value: ${runtime}`); +} + +const dynamoDBClient = new DynamoDBClient({}); + +describe('Idempotency e2e test decorator, default settings', () => { + const testStack = new TestStack({ + stackNameProps: { + stackNamePrefix: RESOURCE_NAME_PREFIX, + testName: 'idempotentDecorator', + }, + }); + + const lambdaFunctionCodeFilePath = join( + __dirname, + 'idempotentDecorator.test.FunctionCode.ts' + ); + + let functionNameDefault: string; + let tableNameDefault: string; + new IdempotencyTestNodejsFunctionAndDynamoTable( + testStack, + { + function: { + entry: lambdaFunctionCodeFilePath, + }, + }, + { + nameSuffix: 'default', + } + ); + + let functionNameDefaultParallel: string; + let tableNameDefaultParallel: string; + new IdempotencyTestNodejsFunctionAndDynamoTable( + testStack, + { + function: { + entry: lambdaFunctionCodeFilePath, + handler: 'handlerParallel', + }, + }, + { + nameSuffix: 'defaultParallel', + } + ); + + let functionNameTimeout: string; + let tableNameTimeout: string; + new IdempotencyTestNodejsFunctionAndDynamoTable( + testStack, + { + function: { + entry: lambdaFunctionCodeFilePath, + handler: 'handlerTimeout', + timeout: Duration.seconds(2), + }, + }, + { + nameSuffix: 'timeout', + } + ); + + let functionNameExpired: string; + let tableNameExpired: string; + new IdempotencyTestNodejsFunctionAndDynamoTable( + testStack, + { + function: { + entry: lambdaFunctionCodeFilePath, + handler: 'handlerExpired', + timeout: Duration.seconds(2), + }, + }, + { + nameSuffix: 'expired', + } + ); + + let functionNameDataIndex: string; + let tableNameDataIndex: string; + new IdempotencyTestNodejsFunctionAndDynamoTable( + testStack, + { + function: { + entry: lambdaFunctionCodeFilePath, + handler: 'handlerWithKeywordArgument', + }, + }, + { + nameSuffix: 'dataIndex', + } + ); + + let functionCustomConfig: string; + let tableNameCustomConfig: string; + new IdempotencyTestNodejsFunctionAndDynamoTable( + testStack, + { + function: { + entry: lambdaFunctionCodeFilePath, + handler: 'handlerCustomized', + }, + table: { + partitionKey: { + name: 'customId', + type: AttributeType.STRING, + }, + }, + }, + { + nameSuffix: 'customConfig', + } + ); + + beforeAll(async () => { + await testStack.deploy(); + + functionNameDefault = testStack.findAndGetStackOutputValue('defaultFn'); + tableNameDefault = testStack.findAndGetStackOutputValue('defaultTable'); + + functionNameDefaultParallel = + testStack.findAndGetStackOutputValue('defaultParallelFn'); + tableNameDefaultParallel = testStack.findAndGetStackOutputValue( + 'defaultParallelTable' + ); + + functionNameTimeout = testStack.findAndGetStackOutputValue('timeoutFn'); + tableNameTimeout = testStack.findAndGetStackOutputValue('timeoutTable'); + + functionNameExpired = testStack.findAndGetStackOutputValue('expiredFn'); + tableNameExpired = testStack.findAndGetStackOutputValue('expiredTable'); + + functionCustomConfig = + testStack.findAndGetStackOutputValue('customConfigFn'); + tableNameCustomConfig = + testStack.findAndGetStackOutputValue('customConfigTable'); + + functionNameDataIndex = testStack.findAndGetStackOutputValue('dataIndexFn'); + tableNameDataIndex = testStack.findAndGetStackOutputValue('dataIndexTable'); + }, SETUP_TIMEOUT); + + test( + 'when called twice with the same payload, it returns the same result and runs the handler once', + async () => { + const payload = { foo: 'bar' }; + + const payloadHash = createHash('md5') + .update(JSON.stringify(payload)) + .digest('base64'); + + const logs = await invokeFunction({ + functionName: functionNameDefault, + times: 2, + invocationMode: 'SEQUENTIAL', + payload: payload, + }); + + const functionLogs = logs.map((log) => log.getFunctionLogs()); + + const idempotencyRecord = await dynamoDBClient.send( + new ScanCommand({ + TableName: tableNameDefault, + }) + ); + expect(idempotencyRecord.Items).toHaveLength(1); + expect(idempotencyRecord.Items?.[0].id).toEqual( + `${functionNameDefault}#${payloadHash}` + ); + expect(idempotencyRecord.Items?.[0].data).toEqual('Hello World'); + expect(idempotencyRecord.Items?.[0].status).toEqual('COMPLETED'); + // During the first invocation the handler should be called, so the logs should contain 1 log + expect(functionLogs[0]).toHaveLength(1); + // We test the content of the log as well as the presence of fields from the context, this + // ensures that the all the arguments are passed to the handler when made idempotent + expect(TestInvocationLogs.parseFunctionLog(functionLogs[0][0])).toEqual( + expect.objectContaining({ + message: 'Got test event: {"foo":"bar"}', + }) + ); + }, + TEST_CASE_TIMEOUT + ); + + test( + 'when called twice in parallel, the handler is called only once', + async () => { + const payload = { foo: 'bar' }; + const payloadHash = createHash('md5') + .update(JSON.stringify(payload)) + .digest('base64'); + const logs = await invokeFunction({ + functionName: functionNameDefaultParallel, + times: 2, + invocationMode: 'PARALLEL', + payload: payload, + }); + + const functionLogs = logs.map((log) => log.getFunctionLogs()); + + const idempotencyRecords = await dynamoDBClient.send( + new ScanCommand({ + TableName: tableNameDefaultParallel, + }) + ); + expect(idempotencyRecords.Items).toHaveLength(1); + expect(idempotencyRecords.Items?.[0].id).toEqual( + `${functionNameDefaultParallel}#${payloadHash}` + ); + expect(idempotencyRecords.Items?.[0].data).toEqual('bar'); + expect(idempotencyRecords.Items?.[0].status).toEqual('COMPLETED'); + expect(idempotencyRecords?.Items?.[0].expiration).toBeGreaterThan( + Date.now() / 1000 + ); + const successfulInvocationLogs = functionLogs.find( + (functionLog) => + functionLog.toString().includes('Processed event') !== undefined + ); + + const failedInvocationLogs = functionLogs.find( + (functionLog) => + functionLog + .toString() + .includes('There is already an execution in progres') !== undefined + ); + + expect(successfulInvocationLogs).toBeDefined(); + expect(failedInvocationLogs).toBeDefined(); + }, + TEST_CASE_TIMEOUT + ); + + test( + 'when the function times out, the second request is processed correctly by the handler', + async () => { + const payload = { foo: 'bar' }; + const payloadHash = createHash('md5') + .update(JSON.stringify(payload.foo)) + .digest('base64'); + + const logs = await invokeFunction({ + functionName: functionNameTimeout, + times: 2, + invocationMode: 'SEQUENTIAL', + payload: Array.from({ length: 2 }, (_, index) => ({ + ...payload, + invocation: index, + })), + }); + const functionLogs = logs.map((log) => log.getFunctionLogs()); + const idempotencyRecord = await dynamoDBClient.send( + new ScanCommand({ + TableName: tableNameTimeout, + }) + ); + expect(idempotencyRecord.Items).toHaveLength(1); + expect(idempotencyRecord.Items?.[0].id).toEqual( + `${functionNameTimeout}#${payloadHash}` + ); + expect(idempotencyRecord.Items?.[0].data).toEqual({ + ...payload, + invocation: 1, + }); + expect(idempotencyRecord.Items?.[0].status).toEqual('COMPLETED'); + + // During the first invocation the handler should be called, so the logs should contain 1 log + expect(functionLogs[0]).toHaveLength(2); + expect(functionLogs[0][0]).toContain('Task timed out after'); + + expect(functionLogs[1]).toHaveLength(1); + expect(TestInvocationLogs.parseFunctionLog(functionLogs[1][0])).toEqual( + expect.objectContaining({ + message: 'Processed event', + details: 'bar', + function_name: functionNameTimeout, + }) + ); + }, + TEST_CASE_TIMEOUT + ); + + test( + 'when the idempotency record is expired, the second request is processed correctly by the handler', + async () => { + const payload = { + foo: 'baz', + }; + const payloadHash = createHash('md5') + .update(JSON.stringify(payload.foo)) + .digest('base64'); + + // Act + const logs = [ + ( + await invokeFunction({ + functionName: functionNameExpired, + times: 1, + invocationMode: 'SEQUENTIAL', + payload: { ...payload, invocation: 0 }, + }) + )[0], + ]; + // Wait for the idempotency record to expire + await new Promise((resolve) => setTimeout(resolve, 2000)); + logs.push( + ( + await invokeFunction({ + functionName: functionNameExpired, + times: 1, + invocationMode: 'SEQUENTIAL', + payload: { ...payload, invocation: 1 }, + }) + )[0] + ); + const functionLogs = logs.map((log) => log.getFunctionLogs()); + + // Assess + const idempotencyRecords = await dynamoDBClient.send( + new ScanCommand({ + TableName: tableNameExpired, + }) + ); + expect(idempotencyRecords.Items).toHaveLength(1); + expect(idempotencyRecords.Items?.[0].id).toEqual( + `${functionNameExpired}#${payloadHash}` + ); + expect(idempotencyRecords.Items?.[0].data).toEqual({ + ...payload, + invocation: 1, + }); + expect(idempotencyRecords.Items?.[0].status).toEqual('COMPLETED'); + + // Both invocations should be successful and the logs should contain 1 log each + expect(functionLogs[0]).toHaveLength(1); + expect(TestInvocationLogs.parseFunctionLog(functionLogs[1][0])).toEqual( + expect.objectContaining({ + message: 'Processed event', + details: 'baz', + function_name: functionNameExpired, + }) + ); + // During the second invocation the handler should be called and complete, so the logs should + // contain 1 log + expect(functionLogs[1]).toHaveLength(1); + expect(TestInvocationLogs.parseFunctionLog(functionLogs[1][0])).toEqual( + expect.objectContaining({ + message: 'Processed event', + details: 'baz', + function_name: functionNameExpired, + }) + ); + }, + TEST_CASE_TIMEOUT + ); + + test( + 'when called with customized function wrapper, it creates ddb entry with custom attributes', + async () => { + const payload = { foo: 'bar' }; + const payloadHash = createHash('md5') + .update(JSON.stringify(payload)) + .digest('base64'); + const logs = await invokeFunction({ + functionName: functionCustomConfig, + times: 1, + invocationMode: 'SEQUENTIAL', + payload: payload, + }); + + const functionLogs = logs.map((log) => log.getFunctionLogs()); + + const idempotencyRecord = await dynamoDBClient.send( + new ScanCommand({ + TableName: tableNameCustomConfig, + }) + ); + expect(idempotencyRecord.Items?.[0]).toStrictEqual({ + customId: `${functionCustomConfig}#${payloadHash}`, + dataAttr: 'bar', + statusAttr: 'COMPLETED', + expiryAttr: expect.any(Number), + inProgressExpiryAttr: expect.any(Number), + }); + + expect(functionLogs[0]).toHaveLength(1); + expect(TestInvocationLogs.parseFunctionLog(functionLogs[0][0])).toEqual( + expect.objectContaining({ + message: 'Processed event', + details: 'bar', + }) + ); + }, + TEST_CASE_TIMEOUT + ); + + test( + 'when called twice for with different payload using data index arugment, it returns the same result and runs the handler once', + async () => { + const payload = [{ id: '1234' }, { id: '5678' }]; + const payloadHash = createHash('md5') + .update(JSON.stringify('bar')) + .digest('base64'); + + const logs = await invokeFunction({ + functionName: functionNameDataIndex, + times: 2, + invocationMode: 'SEQUENTIAL', + payload: payload, + }); + + const functionLogs = logs.map((log) => log.getFunctionLogs()); + + const idempotencyRecord = await dynamoDBClient.send( + new ScanCommand({ + TableName: tableNameDataIndex, + }) + ); + expect(idempotencyRecord.Items).toHaveLength(1); + expect(idempotencyRecord.Items?.[0].id).toEqual( + `${functionNameDataIndex}#${payloadHash}` + ); + expect(idempotencyRecord.Items?.[0].data).toEqual( + 'idempotent result: bar' + ); + expect(idempotencyRecord.Items?.[0].status).toEqual('COMPLETED'); + // During the first invocation the handler should be called, so the logs should contain 1 log + expect(functionLogs[0]).toHaveLength(1); + // We test the content of the log as well as the presence of fields from the context, this + // ensures that the all the arguments are passed to the handler when made idempotent + expect(TestInvocationLogs.parseFunctionLog(functionLogs[0][0])).toEqual( + expect.objectContaining({ + message: 'Got test event', + id: '1234', + foo: 'bar', + }) + ); + }, + TEST_CASE_TIMEOUT + ); + + afterAll(async () => { + if (!process.env.DISABLE_TEARDOWN) { + await testStack.destroy(); + } + }, TEARDOWN_TIMEOUT); +});