From d4e387891614cf266a504a84c5740e2f550f250a Mon Sep 17 00:00:00 2001 From: Alexander Melnyk Date: Sat, 28 Jan 2023 12:11:12 +0100 Subject: [PATCH 1/5] change to more generic name --- ...secretsProvider.class.test.functionCode.ts | 57 +++++++ .../tests/e2e/secretsProvider.class.test.ts | 159 ++++++++++++++++++ .../tests/helpers/cdkAspectGrantAccess.ts | 10 +- 3 files changed, 221 insertions(+), 5 deletions(-) create mode 100644 packages/parameters/tests/e2e/secretsProvider.class.test.functionCode.ts create mode 100644 packages/parameters/tests/e2e/secretsProvider.class.test.ts diff --git a/packages/parameters/tests/e2e/secretsProvider.class.test.functionCode.ts b/packages/parameters/tests/e2e/secretsProvider.class.test.functionCode.ts new file mode 100644 index 0000000000..4e31da5bfc --- /dev/null +++ b/packages/parameters/tests/e2e/secretsProvider.class.test.functionCode.ts @@ -0,0 +1,57 @@ +import { Context } from 'aws-lambda'; +import { SecretsProvider } from '../../lib/secrets'; +import { TinyLogger } from '../helpers/tinyLogger'; +import { transform } from 'esbuild'; +import { SecretsGetOptionsInterface } from '../../lib/types'; + +const logger = new TinyLogger(); +const provider = new SecretsProvider(); + +const secretNamePlain = process.env.SECRET_NAME_PLAIN || ''; +const secretNameObject = process.env.SECRET_NAME_OBJECT || ''; +const secretNameBinary = process.env.SECRET_NAME_BINARY || ''; +const secretNameObjectWithSuffix = process.env.SECRET_NAME_OBJECT_WITH_SUFFIX || ''; +const secretNameBinaryWithSuffix = process.env.SECRET_NAME_BINARY_WITH_SUFFIX || ''; + +const _call_get = async (paramName: string, testName: string, options?: SecretsGetOptionsInterface) : Promise => { + try { + if (!transform) { + const parameterValue = await provider.get(paramName); + logger.log({ + test: testName, + value: parameterValue + }); + } else { + const parameterValue = await provider.get(paramName, options); + logger.log({ + test: testName, + value: parameterValue + }); + } + } catch (err) { + logger.log({ + test: testName, + error: err.message + }); + } +}; + +export const handler = async (_event: unknown, _context: Context): Promise => { + + // Test 1 get single param as plaintext + await _call_get(secretNamePlain, 'get-plain'); + + // Test 2 get single param with transform json + await _call_get(secretNameObject, 'get-transform-json', { transform: 'json' }); + + // Test 3 get single param with transform binary + await _call_get(secretNameBinary, 'get-transform-binary', { transform: 'binary' }); + + // Test 4 get single param with transform auto json + await _call_get(secretNameObjectWithSuffix, 'get-transform-auto-json', { transform: 'auto' }); + + // Test 5 get single param with transform auto binary + await _call_get(secretNameBinaryWithSuffix, 'get-transform-auto-binary', { transform: 'auto' }); + +}; + diff --git a/packages/parameters/tests/e2e/secretsProvider.class.test.ts b/packages/parameters/tests/e2e/secretsProvider.class.test.ts new file mode 100644 index 0000000000..af65632e03 --- /dev/null +++ b/packages/parameters/tests/e2e/secretsProvider.class.test.ts @@ -0,0 +1,159 @@ +/** + * Test SecretsPorovider class + * + * @group e2e/parameters/secrets/class + */ +import { + createStackWithLambdaFunction, + generateUniqueName, + invokeFunction, + isValidRuntimeKey +} from '../../../commons/tests/utils/e2eUtils'; +import { RESOURCE_NAME_PREFIX, SETUP_TIMEOUT, TEARDOWN_TIMEOUT, TEST_CASE_TIMEOUT } from './constants'; +import { v4 } from 'uuid'; +import { Tracing } from 'aws-cdk-lib/aws-lambda'; +import { deployStack, destroyStack } from '../../../commons/tests/utils/cdk-cli'; +import { App, Aspects, SecretValue, Stack } from 'aws-cdk-lib'; +import path from 'path'; +import { Secret } from 'aws-cdk-lib/aws-secretsmanager'; +import { InvocationLogs } from '../../../commons/tests/utils/InvocationLogs'; +import { ResourceAccessGranter } from '../helpers/cdkAspectGrantAccess'; + +const runtime: string = process.env.RUNTIME || 'nodejs18x'; + +if (!isValidRuntimeKey(runtime)) { + throw new Error(`Invalid runtime key: ${runtime}`); +} + +const uuid = v4(); +const stackName = generateUniqueName(RESOURCE_NAME_PREFIX, uuid, runtime, 'secretsProvider'); +const functionName = generateUniqueName(RESOURCE_NAME_PREFIX, uuid, runtime, 'secretsProvider'); +const lambdaFunctionCodeFile = 'secretsProvider.class.test.functionCode.ts'; + +const secretNamePlain = generateUniqueName(RESOURCE_NAME_PREFIX, uuid, runtime, 'testSecretPlain'); +const secretNameObject = generateUniqueName(RESOURCE_NAME_PREFIX, uuid, runtime, 'testSecretObject'); +const secretNameBinary = generateUniqueName(RESOURCE_NAME_PREFIX, uuid, runtime, 'testSecretBinary'); +const secretNameObjectWithSuffix = generateUniqueName(RESOURCE_NAME_PREFIX, uuid, runtime, 'testSecretObject.json'); +const secretNameBinaryWithSuffix = generateUniqueName(RESOURCE_NAME_PREFIX, uuid, runtime, 'testSecretObject.binary'); + +const invocationCount = 1; + +const integTestApp = new App(); +let stack: Stack; + +describe(`parameters E2E tests (SecretsProvider) for runtime: ${runtime}`, () => { + + let invocationLogs: InvocationLogs[]; + + beforeAll(async () => { + stack = createStackWithLambdaFunction({ + app: integTestApp, + stackName: stackName, + functionName: functionName, + functionEntry: path.join(__dirname, lambdaFunctionCodeFile), + tracing: Tracing.ACTIVE, + environment: { + UUID: uuid, + SECRET_NAME_PLAIN: secretNamePlain, + SECRET_NAME_OBJECT: secretNameObject, + SECRET_NAME_BINARY: secretNameBinary, + SECRET_NAME_OBJECT_WITH_SUFFIX: secretNameObjectWithSuffix, + SECRET_NAME_BINARY_WITH_SUFFIX: secretNameBinaryWithSuffix + }, + runtime: runtime + }); + const secretString = new Secret(stack, 'testSecretPlain', { + secretName: secretNamePlain, + secretStringValue: SecretValue.unsafePlainText('foo') + }); + + const secretObject = new Secret(stack, 'testSecretObject', { + secretName: secretNameObject, + secretObjectValue: { + foo: SecretValue.unsafePlainText('bar'), + } + }); + + const secretBinary = new Secret(stack, 'testSecretBinary', { + secretName: secretNameBinary, + secretStringValue: SecretValue.unsafePlainText('Zm9v') // 'foo' encoded in base64 + }); + + const secretObjectWithSuffix = new Secret(stack, 'testSecretObjectWithSuffix', { + secretName: secretNameObjectWithSuffix, + secretObjectValue: { + foo: SecretValue.unsafePlainText('bar') + } + }); + + const secretBinaryWithSuffix = new Secret(stack, 'testSecretBinaryWithSuffix', { + secretName: secretNameBinaryWithSuffix, + secretStringValue: SecretValue.unsafePlainText('Zm9v') // 'foo' encoded in base64 + }); + + Aspects.of(stack).add(new ResourceAccessGranter([ secretString, secretObject, secretBinary, secretObjectWithSuffix, secretBinaryWithSuffix ])); + await deployStack(integTestApp, stack); + + invocationLogs = await invokeFunction(functionName, invocationCount, 'SEQUENTIAL'); + + }, SETUP_TIMEOUT); + + describe('SecretsProvider usage', () => { + it('should retrieve a single parameter', async () => { + + const logs = invocationLogs[0].getFunctionLogs(); + const testLog = InvocationLogs.parseFunctionLog(logs[0]); + + expect(testLog).toStrictEqual({ + test: 'get-plain', + value: 'foo' + }); + }, TEST_CASE_TIMEOUT); + + it('should retrieve a single parameter with transform json', async () => { + const logs = invocationLogs[0].getFunctionLogs(); + const testLog = InvocationLogs.parseFunctionLog(logs[1]); + + expect(testLog).toStrictEqual({ + test: 'get-transform-json', + value: { foo: 'bar' } + }); + }, TEST_CASE_TIMEOUT); + + it('should retrieve single param with transform binary', async () => { + const logs = invocationLogs[0].getFunctionLogs(); + const testLog = InvocationLogs.parseFunctionLog(logs[2]); + + expect(testLog).toStrictEqual({ + test: 'get-transform-binary', + value: 'foo' + }); + }, TEST_CASE_TIMEOUT); + }); + + it('should retrieve single param with transform auto json', async () => { + const logs = invocationLogs[0].getFunctionLogs(); + const testLog = InvocationLogs.parseFunctionLog(logs[3]); + + expect(testLog).toStrictEqual({ + test: 'get-transform-auto-json', + value: { foo: 'bar' } + }); + }, TEST_CASE_TIMEOUT); + + it('should retrieve single param wit transform auto binary', async () => { + const logs = invocationLogs[0].getFunctionLogs(); + const testLog = InvocationLogs.parseFunctionLog(logs[4]); + + expect(testLog).toStrictEqual({ + test: 'get-transform-auto-binary', + value: 'foo' + }); + }); + + afterAll(async () => { + if (!process.env.DISABLE_TEARDOWN) { + await destroyStack(integTestApp, stack); + } + }, TEARDOWN_TIMEOUT); +}); \ No newline at end of file diff --git a/packages/parameters/tests/helpers/cdkAspectGrantAccess.ts b/packages/parameters/tests/helpers/cdkAspectGrantAccess.ts index 22f8c09e6f..57d99a393b 100644 --- a/packages/parameters/tests/helpers/cdkAspectGrantAccess.ts +++ b/packages/parameters/tests/helpers/cdkAspectGrantAccess.ts @@ -6,23 +6,23 @@ import { Secret } from 'aws-cdk-lib/aws-secretsmanager'; /** * An aspect that grants access to resources to a Lambda function. - * + * * In our integration tests, we dynamically generate AWS CDK stacks that contain a Lambda function. * We want to grant access to resources to the Lambda function, but we don't know the name of the * Lambda function at the time we create the resources. Additionally, we want to keep the code * that creates the stacks and functions as generic as possible. - * + * * This aspect allows us to grant access to specific resources to all Lambda functions in a stack * after the stack tree has been generated and before the stack is deployed. This aspect is * used to grant access to different resource types (DynamoDB tables, SSM parameters, etc.). - * + * * @see {@link https://docs.aws.amazon.com/cdk/v2/guide/aspects.html|CDK Docs - Aspects} */ export class ResourceAccessGranter implements IAspect { private readonly resources: Table[] | Secret[]; - public constructor(tables: Table[] | Secret[]) { - this.resources = tables; + public constructor(resources: Table[] | Secret[]) { + this.resources = resources; } public visit(node: IConstruct): void { From 1eb81d47d75270f5e48e42d765a4db1b32b440e0 Mon Sep 17 00:00:00 2001 From: Alexander Melnyk Date: Wed, 1 Feb 2023 10:33:51 +0100 Subject: [PATCH 2/5] fix conditional --- .../tests/e2e/secretsProvider.class.test.functionCode.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/parameters/tests/e2e/secretsProvider.class.test.functionCode.ts b/packages/parameters/tests/e2e/secretsProvider.class.test.functionCode.ts index 4e31da5bfc..c857e50fa2 100644 --- a/packages/parameters/tests/e2e/secretsProvider.class.test.functionCode.ts +++ b/packages/parameters/tests/e2e/secretsProvider.class.test.functionCode.ts @@ -1,7 +1,6 @@ import { Context } from 'aws-lambda'; import { SecretsProvider } from '../../lib/secrets'; import { TinyLogger } from '../helpers/tinyLogger'; -import { transform } from 'esbuild'; import { SecretsGetOptionsInterface } from '../../lib/types'; const logger = new TinyLogger(); @@ -15,7 +14,7 @@ const secretNameBinaryWithSuffix = process.env.SECRET_NAME_BINARY_WITH_SUFFIX || const _call_get = async (paramName: string, testName: string, options?: SecretsGetOptionsInterface) : Promise => { try { - if (!transform) { + if (!options) { const parameterValue = await provider.get(paramName); logger.log({ test: testName, From b86860490ec3b34d9e2c5c954794f144a1a84547 Mon Sep 17 00:00:00 2001 From: Alexander Melnyk Date: Wed, 1 Feb 2023 10:52:05 +0100 Subject: [PATCH 3/5] add provider to signature --- ...secretsProvider.class.test.functionCode.ts | 27 +++++++++---------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/packages/parameters/tests/e2e/secretsProvider.class.test.functionCode.ts b/packages/parameters/tests/e2e/secretsProvider.class.test.functionCode.ts index c857e50fa2..0c1ed59af8 100644 --- a/packages/parameters/tests/e2e/secretsProvider.class.test.functionCode.ts +++ b/packages/parameters/tests/e2e/secretsProvider.class.test.functionCode.ts @@ -4,7 +4,7 @@ import { TinyLogger } from '../helpers/tinyLogger'; import { SecretsGetOptionsInterface } from '../../lib/types'; const logger = new TinyLogger(); -const provider = new SecretsProvider(); +const defaultProvider = new SecretsProvider(); const secretNamePlain = process.env.SECRET_NAME_PLAIN || ''; const secretNameObject = process.env.SECRET_NAME_OBJECT || ''; @@ -12,21 +12,16 @@ const secretNameBinary = process.env.SECRET_NAME_BINARY || ''; const secretNameObjectWithSuffix = process.env.SECRET_NAME_OBJECT_WITH_SUFFIX || ''; const secretNameBinaryWithSuffix = process.env.SECRET_NAME_BINARY_WITH_SUFFIX || ''; -const _call_get = async (paramName: string, testName: string, options?: SecretsGetOptionsInterface) : Promise => { +const _call_get = async (paramName: string, testName: string, options?: SecretsGetOptionsInterface, provider?: SecretsProvider,): Promise => { try { - if (!options) { - const parameterValue = await provider.get(paramName); - logger.log({ - test: testName, - value: parameterValue - }); - } else { - const parameterValue = await provider.get(paramName, options); - logger.log({ - test: testName, - value: parameterValue - }); - } + // we might get a provider with specific sdk options, otherwise fallback to default + const currentProvider = provider ? provider : defaultProvider; + + const parameterValue = await currentProvider.get(paramName, options); + logger.log({ + test: testName, + value: parameterValue + }); } catch (err) { logger.log({ test: testName, @@ -52,5 +47,7 @@ export const handler = async (_event: unknown, _context: Context): Promise // Test 5 get single param with transform auto binary await _call_get(secretNameBinaryWithSuffix, 'get-transform-auto-binary', { transform: 'auto' }); + // TOOD: add more tests once we can pass sdk configuration to the provider + }; From 4e778309ea68cd5b9f33bcb1b4f6e18f7c053396 Mon Sep 17 00:00:00 2001 From: Alexander Melnyk Date: Wed, 1 Feb 2023 10:53:04 +0100 Subject: [PATCH 4/5] add github issue to todo --- .../tests/e2e/secretsProvider.class.test.functionCode.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/parameters/tests/e2e/secretsProvider.class.test.functionCode.ts b/packages/parameters/tests/e2e/secretsProvider.class.test.functionCode.ts index 0c1ed59af8..d42772a15d 100644 --- a/packages/parameters/tests/e2e/secretsProvider.class.test.functionCode.ts +++ b/packages/parameters/tests/e2e/secretsProvider.class.test.functionCode.ts @@ -47,7 +47,7 @@ export const handler = async (_event: unknown, _context: Context): Promise // Test 5 get single param with transform auto binary await _call_get(secretNameBinaryWithSuffix, 'get-transform-auto-binary', { transform: 'auto' }); - // TOOD: add more tests once we can pass sdk configuration to the provider + // TOOD: add more tests once we can pass sdk configuration to the provider, after #1222 closed }; From 03f8218bd7ce8cc265b289e5b4ac1f11f7366c55 Mon Sep 17 00:00:00 2001 From: Alexander Melnyk Date: Tue, 7 Feb 2023 14:11:47 +0100 Subject: [PATCH 5/5] add tests for caching and forceFetch --- ...secretsProvider.class.test.functionCode.ts | 44 ++++++++++++++++++- .../tests/e2e/secretsProvider.class.test.ts | 33 +++++++++++++- 2 files changed, 73 insertions(+), 4 deletions(-) diff --git a/packages/parameters/tests/e2e/secretsProvider.class.test.functionCode.ts b/packages/parameters/tests/e2e/secretsProvider.class.test.functionCode.ts index d42772a15d..655b9472a6 100644 --- a/packages/parameters/tests/e2e/secretsProvider.class.test.functionCode.ts +++ b/packages/parameters/tests/e2e/secretsProvider.class.test.functionCode.ts @@ -2,6 +2,8 @@ import { Context } from 'aws-lambda'; import { SecretsProvider } from '../../lib/secrets'; import { TinyLogger } from '../helpers/tinyLogger'; import { SecretsGetOptionsInterface } from '../../lib/types'; +import { SecretsManagerClient } from '@aws-sdk/client-secrets-manager'; +import { middleware } from '../helpers/sdkMiddlewareRequestCounter'; const logger = new TinyLogger(); const defaultProvider = new SecretsProvider(); @@ -11,6 +13,14 @@ const secretNameObject = process.env.SECRET_NAME_OBJECT || ''; const secretNameBinary = process.env.SECRET_NAME_BINARY || ''; const secretNameObjectWithSuffix = process.env.SECRET_NAME_OBJECT_WITH_SUFFIX || ''; const secretNameBinaryWithSuffix = process.env.SECRET_NAME_BINARY_WITH_SUFFIX || ''; +const secretNamePlainChached = process.env.SECRET_NAME_PLAIN_CACHED || ''; + +// Provider test 8, 9 +const customClient = new SecretsManagerClient({}); +customClient.middlewareStack.use(middleware); +const providerWithMiddleware = new SecretsProvider({ + awsSdkV3Client: customClient +}); const _call_get = async (paramName: string, testName: string, options?: SecretsGetOptionsInterface, provider?: SecretsProvider,): Promise => { try { @@ -47,7 +57,37 @@ export const handler = async (_event: unknown, _context: Context): Promise // Test 5 get single param with transform auto binary await _call_get(secretNameBinaryWithSuffix, 'get-transform-auto-binary', { transform: 'auto' }); - // TOOD: add more tests once we can pass sdk configuration to the provider, after #1222 closed + // Test 6 + // get parameter twice with middleware, which counts number of SDK requests, we check later if we only called SecretManager API once + try { + middleware.counter = 0; + await providerWithMiddleware.get(secretNamePlainChached); + await providerWithMiddleware.get(secretNamePlainChached); + logger.log({ + test: 'get-plain-cached', + value: middleware.counter // should be 1 + }); + } catch (err) { + logger.log({ + test: secretNamePlainChached, + error: err.message + }); + } + // Test 7 + // get parameter twice, but force fetch 2nd time, we count number of SDK requests and check that we made two API calls + try { + middleware.counter = 0; + await providerWithMiddleware.get(secretNamePlainChached); + await providerWithMiddleware.get(secretNamePlainChached, { forceFetch: true }); + logger.log({ + test: 'get-plain-force', + value: middleware.counter // should be 2 + }); + } catch (err) { + logger.log({ + test: secretNamePlainChached, + error: err.message + }); + } }; - diff --git a/packages/parameters/tests/e2e/secretsProvider.class.test.ts b/packages/parameters/tests/e2e/secretsProvider.class.test.ts index af65632e03..a4ea351151 100644 --- a/packages/parameters/tests/e2e/secretsProvider.class.test.ts +++ b/packages/parameters/tests/e2e/secretsProvider.class.test.ts @@ -31,6 +31,7 @@ const functionName = generateUniqueName(RESOURCE_NAME_PREFIX, uuid, runtime, 'se const lambdaFunctionCodeFile = 'secretsProvider.class.test.functionCode.ts'; const secretNamePlain = generateUniqueName(RESOURCE_NAME_PREFIX, uuid, runtime, 'testSecretPlain'); +const secretNamePlainCached = generateUniqueName(RESOURCE_NAME_PREFIX, uuid, runtime, 'testSecretPlainCached'); const secretNameObject = generateUniqueName(RESOURCE_NAME_PREFIX, uuid, runtime, 'testSecretObject'); const secretNameBinary = generateUniqueName(RESOURCE_NAME_PREFIX, uuid, runtime, 'testSecretBinary'); const secretNameObjectWithSuffix = generateUniqueName(RESOURCE_NAME_PREFIX, uuid, runtime, 'testSecretObject.json'); @@ -58,10 +59,12 @@ describe(`parameters E2E tests (SecretsProvider) for runtime: ${runtime}`, () => SECRET_NAME_OBJECT: secretNameObject, SECRET_NAME_BINARY: secretNameBinary, SECRET_NAME_OBJECT_WITH_SUFFIX: secretNameObjectWithSuffix, - SECRET_NAME_BINARY_WITH_SUFFIX: secretNameBinaryWithSuffix + SECRET_NAME_BINARY_WITH_SUFFIX: secretNameBinaryWithSuffix, + SECRET_NAME_PLAIN_CACHED: secretNamePlainCached, }, runtime: runtime }); + const secretString = new Secret(stack, 'testSecretPlain', { secretName: secretNamePlain, secretStringValue: SecretValue.unsafePlainText('foo') @@ -91,7 +94,13 @@ describe(`parameters E2E tests (SecretsProvider) for runtime: ${runtime}`, () => secretStringValue: SecretValue.unsafePlainText('Zm9v') // 'foo' encoded in base64 }); - Aspects.of(stack).add(new ResourceAccessGranter([ secretString, secretObject, secretBinary, secretObjectWithSuffix, secretBinaryWithSuffix ])); + const secretStringCached = new Secret(stack, 'testSecretStringCached', { + secretName: secretNamePlainCached, + secretStringValue: SecretValue.unsafePlainText('foo') + }); + + Aspects.of(stack).add(new ResourceAccessGranter([ secretString, secretObject, secretBinary, secretObjectWithSuffix, secretBinaryWithSuffix, secretStringCached ])); + await deployStack(integTestApp, stack); invocationLogs = await invokeFunction(functionName, invocationCount, 'SEQUENTIAL'); @@ -151,6 +160,26 @@ describe(`parameters E2E tests (SecretsProvider) for runtime: ${runtime}`, () => }); }); + it('should retrieve single parameter cached', async () => { + const logs = invocationLogs[0].getFunctionLogs(); + const testLogFirst = InvocationLogs.parseFunctionLog(logs[5]); + + expect(testLogFirst).toStrictEqual({ + test: 'get-plain-cached', + value: 1 + }); + }); + + it('should retrieve single parameter twice without caching', async () => { + const logs = invocationLogs[0].getFunctionLogs(); + const testLogFirst = InvocationLogs.parseFunctionLog(logs[6]); + + expect(testLogFirst).toStrictEqual({ + test: 'get-plain-force', + value: 1 + }); + }); + afterAll(async () => { if (!process.env.DISABLE_TEARDOWN) { await destroyStack(integTestApp, stack);