Skip to content

fix(parameters): handle base64/binaries in transformer #1326

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Feb 22, 2023
Merged

fix(parameters): handle base64/binaries in transformer #1326

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 9 additions & 7 deletions packages/parameters/src/BaseProvider.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -131,6 +131,11 @@ abstract class BaseProvider implements BaseProviderInterface {
const transformValue = (value: string | Uint8Array | undefined, transform: TransformOptions, throwOnTransformError: boolean, key: string): string | Record<string, unknown> | undefined => {
try {
const normalizedTransform = transform.toLowerCase();

if (value instanceof Uint8Array) {
value = new TextDecoder('utf-8').decode(value);
}

if (
(normalizedTransform === TRANSFORM_METHOD_JSON ||
(normalizedTransform === 'auto' && key.toLowerCase().endsWith(`.${TRANSFORM_METHOD_JSON}`))) &&
Expand All @@ -139,15 +144,12 @@ const transformValue = (value: string | Uint8Array | undefined, transform: Trans
return JSON.parse(value) as Record<string, unknown>;
} else if (
(normalizedTransform === TRANSFORM_METHOD_BINARY ||
(normalizedTransform === 'auto' && key.toLowerCase().endsWith(`.${TRANSFORM_METHOD_BINARY}`)))
(normalizedTransform === 'auto' && key.toLowerCase().endsWith(`.${TRANSFORM_METHOD_BINARY}`))) &&
typeof value === 'string'
) {
if (typeof value === 'string') {
return new TextDecoder('utf-8').decode(fromBase64(value));
} else {
return new TextDecoder('utf-8').decode(value);
}
return new TextDecoder('utf-8').decode(fromBase64(value));
} else {
return value as string;
return value;
}
} catch (error) {
if (throwOnTransformError)
Expand Down
30 changes: 13 additions & 17 deletions packages/parameters/tests/e2e/appConfigProvider.class.test.functionCode.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,7 @@ const application = process.env.APPLICATION_NAME || 'my-app';
const environment = process.env.ENVIRONMENT_NAME || 'my-env';
const freeFormJsonName = process.env.FREEFORM_JSON_NAME || 'freeform-json';
const freeFormYamlName = process.env.FREEFORM_YAML_NAME || 'freeform-yaml';
const freeFormPlainTextNameA = process.env.FREEFORM_PLAIN_TEXT_NAME_A || 'freeform-plain-text';
const freeFormPlainTextNameB = process.env.FREEFORM_PLAIN_TEXT_NAME_B || 'freeform-plain-text';
const freeFormBase64encodedPlainText = process.env.FREEFORM_BASE64_ENCODED_PLAIN_TEXT_NAME || 'freeform-plain-text';
const featureFlagName = process.env.FEATURE_FLAG_NAME || 'feature-flag';

const defaultProvider = new AppConfigProvider({
Expand Down Expand Up @@ -65,28 +64,25 @@ const _call_get = async (
};

export const handler = async (_event: unknown, _context: Context): Promise<void> => {
// Test 1 - get a single parameter as-is (no transformation)
await _call_get(freeFormPlainTextNameA, 'get');
// Test 1 - get a single parameter as-is (no transformation - should return an Uint8Array)
await _call_get(freeFormYamlName, 'get');

// Test 2 - get a free-form JSON and apply binary transformation (should return a stringified JSON)
await _call_get(freeFormJsonName, 'get-freeform-json-binary', { transform: 'binary' });
// Test 2 - get a free-form JSON and apply json transformation (should return an object)
await _call_get(freeFormJsonName, 'get-freeform-json-binary', { transform: 'json' });

// Test 3 - get a free-form YAML and apply binary transformation (should return a string-encoded YAML)
await _call_get(freeFormYamlName, 'get-freeform-yaml-binary', { transform: 'binary' });
// Test 3 - get a free-form base64-encoded plain text and apply binary transformation (should return a decoded string)
await _call_get(freeFormBase64encodedPlainText, 'get-freeform-base64-plaintext-binary', { transform: 'binary' });

// Test 4 - get a free-form plain text and apply binary transformation (should return a string)
await _call_get(freeFormPlainTextNameB, 'get-freeform-plain-text-binary', { transform: 'binary' });

// Test 5 - get a feature flag and apply binary transformation (should return a stringified JSON)
await _call_get(featureFlagName, 'get-feature-flag-binary', { transform: 'binary' });
// Test 5 - get a feature flag and apply json transformation (should return an object)
await _call_get(featureFlagName, 'get-feature-flag-binary', { transform: 'json' });

// Test 6
// get parameter twice with middleware, which counts the number of requests, we check later if we only called AppConfig API once
try {
providerWithMiddleware.clearCache();
middleware.counter = 0;
await providerWithMiddleware.get(freeFormPlainTextNameA);
await providerWithMiddleware.get(freeFormPlainTextNameA);
await providerWithMiddleware.get(freeFormBase64encodedPlainText);
await providerWithMiddleware.get(freeFormBase64encodedPlainText);
logger.log({
test: 'get-cached',
value: middleware.counter // should be 1
Expand All @@ -103,8 +99,8 @@ export const handler = async (_event: unknown, _context: Context): Promise<void>
try {
providerWithMiddleware.clearCache();
middleware.counter = 0;
await providerWithMiddleware.get(freeFormPlainTextNameA);
await providerWithMiddleware.get(freeFormPlainTextNameA, { forceFetch: true });
await providerWithMiddleware.get(freeFormBase64encodedPlainText);
await providerWithMiddleware.get(freeFormBase64encodedPlainText, { forceFetch: true });
logger.log({
test: 'get-forced',
value: middleware.counter // should be 2
Expand Down
103 changes: 34 additions & 69 deletions packages/parameters/tests/e2e/appConfigProvider.class.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
*/
import path from 'path';
import { App, Stack, Aspects } from 'aws-cdk-lib';
import { toBase64 } from '@aws-sdk/util-base64-node';
import { v4 } from 'uuid';
import {
generateUniqueName,
Expand Down Expand Up @@ -44,8 +45,7 @@ const environmentName = generateUniqueName(RESOURCE_NAME_PREFIX, uuid, runtime,
const deploymentStrategyName = generateUniqueName(RESOURCE_NAME_PREFIX, uuid, runtime, 'immediate');
const freeFormJsonName = generateUniqueName(RESOURCE_NAME_PREFIX, uuid, runtime, 'freeFormJson');
const freeFormYamlName = generateUniqueName(RESOURCE_NAME_PREFIX, uuid, runtime, 'freeFormYaml');
const freeFormPlainTextNameA = generateUniqueName(RESOURCE_NAME_PREFIX, uuid, runtime, 'freeFormPlainTextA');
const freeFormPlainTextNameB = generateUniqueName(RESOURCE_NAME_PREFIX, uuid, runtime, 'freeFormPlainTextB');
const freeFormBase64PlainTextName = generateUniqueName(RESOURCE_NAME_PREFIX, uuid, runtime, 'freeFormBase64PlainText');
const featureFlagName = generateUniqueName(RESOURCE_NAME_PREFIX, uuid, runtime, 'featureFlag');

const freeFormJsonValue = {
Expand Down Expand Up @@ -85,33 +85,30 @@ let stack: Stack;
* The parameters created are:
* - Free-form JSON
* - Free-form YAML
* - 2x Free-form plain text
* - Free-form plain text base64-encoded string
* - Feature flag
*
* These parameters allow to retrieve the values and test some transformations.
*
* The tests are:
*
* Test 1
* get a single parameter as-is (no transformation)
* get a single parameter as-is (no transformation - should return an Uint8Array)
*
* Test 2
* get a free-form JSON and apply binary transformation (should return a stringified JSON)
* get a free-form JSON and apply json transformation (should return an object)
*
* Test 3
* get a free-form YAML and apply binary transformation (should return a string-encoded YAML)
* get a free-form base64-encoded plain text and apply binary transformation (should return a decoded string)
*
* Test 4
* get a free-form plain text and apply binary transformation (should return a string)
* get a feature flag and apply json transformation (should return an object)
*
* Test 5
* get a feature flag and apply binary transformation (should return a stringified JSON)
*
* Test 6
* get parameter twice with middleware, which counts the number of requests,
* we check later if we only called AppConfig API once
*
* Test 7
* Test 6
* get parameter twice, but force fetch 2nd time, we count number of SDK requests and
* check that we made two API calls
*
Expand Down Expand Up @@ -140,8 +137,7 @@ describe(`parameters E2E tests (appConfigProvider) for runtime ${runtime}`, () =
ENVIRONMENT_NAME: environmentName,
FREEFORM_JSON_NAME: freeFormJsonName,
FREEFORM_YAML_NAME: freeFormYamlName,
FREEFORM_PLAIN_TEXT_NAME_A: freeFormPlainTextNameA,
FREEFORM_PLAIN_TEXT_NAME_B: freeFormPlainTextNameB,
FREEFORM_BASE64_ENCODED_PLAIN_TEXT_NAME: freeFormBase64PlainTextName,
FEATURE_FLAG_NAME: featureFlagName,
},
runtime,
Expand Down Expand Up @@ -187,33 +183,19 @@ describe(`parameters E2E tests (appConfigProvider) for runtime ${runtime}`, () =
});
freeFormYaml.node.addDependency(freeFormJson);

const freeFormPlainTextA = createAppConfigConfigurationProfile({
const freeFormBase64PlainText = createAppConfigConfigurationProfile({
stack,
application,
environment,
deploymentStrategy,
name: freeFormPlainTextNameA,
name: freeFormBase64PlainTextName,
type: 'AWS.Freeform',
content: {
content: freeFormPlainTextValue,
content: toBase64(new TextEncoder().encode(freeFormPlainTextValue)),
contentType: 'text/plain',
}
});
freeFormPlainTextA.node.addDependency(freeFormYaml);

const freeFormPlainTextB = createAppConfigConfigurationProfile({
stack,
application,
environment,
deploymentStrategy,
name: freeFormPlainTextNameB,
type: 'AWS.Freeform',
content: {
content: freeFormPlainTextValue,
contentType: 'text/plain',
}
});
freeFormPlainTextB.node.addDependency(freeFormPlainTextA);
freeFormBase64PlainText.node.addDependency(freeFormYaml);

const featureFlag = createAppConfigConfigurationProfile({
stack,
Expand All @@ -227,14 +209,13 @@ describe(`parameters E2E tests (appConfigProvider) for runtime ${runtime}`, () =
contentType: 'application/json',
}
});
featureFlag.node.addDependency(freeFormPlainTextB);
featureFlag.node.addDependency(freeFormBase64PlainText);

// Grant access to the Lambda function to the AppConfig resources.
Aspects.of(stack).add(new ResourceAccessGranter([
freeFormJson,
freeFormYaml,
freeFormPlainTextA,
freeFormPlainTextB,
freeFormBase64PlainText,
featureFlag,
]));

Expand All @@ -248,8 +229,8 @@ describe(`parameters E2E tests (appConfigProvider) for runtime ${runtime}`, () =

describe('AppConfigProvider usage', () => {

// Test 1 - get a single parameter as-is (no transformation)
it('should retrieve single parameter', () => {
// Test 1 - get a single parameter as-is (no transformation - should return an Uint8Array)
it('should retrieve single parameter as-is', () => {

const logs = invocationLogs[0].getFunctionLogs();
const testLog = InvocationLogs.parseFunctionLog(logs[0]);
Expand All @@ -258,75 +239,59 @@ describe(`parameters E2E tests (appConfigProvider) for runtime ${runtime}`, () =
test: 'get',
value: JSON.parse(
JSON.stringify(
encoder.encode(freeFormPlainTextValue)
encoder.encode(freeFormYamlValue)
)
),
});

});

// Test 2 - get a free-form JSON and apply binary transformation
// (should return a stringified JSON)
it('should retrieve single free-form JSON parameter with binary transformation', () => {
// Test 2 - get a free-form JSON and apply json transformation (should return an object)
it('should retrieve a free-form JSON parameter with JSON transformation', () => {

const logs = invocationLogs[0].getFunctionLogs();
const testLog = InvocationLogs.parseFunctionLog(logs[1]);

expect(testLog).toStrictEqual({
test: 'get-freeform-json-binary',
value: JSON.stringify(freeFormJsonValue),
value: freeFormJsonValue,
});

});

// Test 3 - get a free-form YAML and apply binary transformation
// (should return a string-encoded YAML)
it('should retrieve single free-form YAML parameter with binary transformation', () => {
// Test 3 - get a free-form base64-encoded plain text and apply binary transformation
// (should return a decoded string)
it('should retrieve a base64-encoded plain text parameter with binary transformation', () => {

const logs = invocationLogs[0].getFunctionLogs();
const testLog = InvocationLogs.parseFunctionLog(logs[2]);

expect(testLog).toStrictEqual({
test: 'get-freeform-yaml-binary',
value: freeFormYamlValue,
});

});

// Test 4 - get a free-form plain text and apply binary transformation
// (should return a string)
it('should retrieve single free-form plain text parameter with binary transformation', () => {

const logs = invocationLogs[0].getFunctionLogs();
const testLog = InvocationLogs.parseFunctionLog(logs[3]);

expect(testLog).toStrictEqual({
test: 'get-freeform-plain-text-binary',
test: 'get-freeform-base64-plaintext-binary',
value: freeFormPlainTextValue,
});

});

// Test 5 - get a feature flag and apply binary transformation
// (should return a stringified JSON)
it('should retrieve single feature flag parameter with binary transformation', () => {
// Test 4 - get a feature flag and apply json transformation (should return an object)
it('should retrieve a feature flag parameter with JSON transformation', () => {

const logs = invocationLogs[0].getFunctionLogs();
const testLog = InvocationLogs.parseFunctionLog(logs[4]);
const testLog = InvocationLogs.parseFunctionLog(logs[3]);

expect(testLog).toStrictEqual({
test: 'get-feature-flag-binary',
value: JSON.stringify(featureFlagValue.values),
value: featureFlagValue.values,
});

});

// Test 6 - get parameter twice with middleware, which counts the number
// Test 5 - get parameter twice with middleware, which counts the number
// of requests, we check later if we only called AppConfig API once
it('should retrieve single parameter cached', () => {

const logs = invocationLogs[0].getFunctionLogs();
const testLog = InvocationLogs.parseFunctionLog(logs[5]);
const testLog = InvocationLogs.parseFunctionLog(logs[4]);

expect(testLog).toStrictEqual({
test: 'get-cached',
Expand All @@ -335,12 +300,12 @@ describe(`parameters E2E tests (appConfigProvider) for runtime ${runtime}`, () =

}, TEST_CASE_TIMEOUT);

// Test 7 - get parameter twice, but force fetch 2nd time,
// Test 6 - get parameter twice, but force fetch 2nd time,
// we count number of SDK requests and check that we made two API calls
it('should retrieve single parameter twice without caching', async () => {

const logs = invocationLogs[0].getFunctionLogs();
const testLog = InvocationLogs.parseFunctionLog(logs[6]);
const testLog = InvocationLogs.parseFunctionLog(logs[5]);

expect(testLog).toStrictEqual({
test: 'get-forced',
Expand Down
10 changes: 3 additions & 7 deletions packages/parameters/tests/unit/BaseProvider.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -206,19 +206,15 @@ describe('Class: BaseProvider', () => {

});

test('when called with a binary transform, and the value is a valid binary, it returns the decoded value', async () => {
test('when called with a binary transform, and the value is a valid binary but NOT base64 encoded, it throws', async () => {

// Prepare
const mockData = encoder.encode('my-value');
const provider = new TestProvider();
jest.spyOn(provider, '_get').mockImplementation(() => new Promise((resolve, _reject) => resolve(mockData as unknown as string)));

// Act
const value = await provider.get('my-parameter', { transform: 'binary' });

// Assess
expect(typeof value).toBe('string');
expect(value).toEqual('my-value');
// Act & Assess
await expect(provider.get('my-parameter', { transform: 'binary' })).rejects.toThrowError(TransformParameterError);

});

Expand Down
8 changes: 4 additions & 4 deletions packages/parameters/tests/unit/getAppConfig.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,11 +16,11 @@ import {
import { mockClient } from 'aws-sdk-client-mock';
import 'aws-sdk-client-mock-jest';
import type { GetAppConfigCombinedInterface } from '../../src/types/AppConfigProvider';
import { toBase64 } from '@aws-sdk/util-base64-node';

describe('Function: getAppConfig', () => {
const client = mockClient(AppConfigDataClient);
const encoder = new TextEncoder();
const decoder = new TextDecoder();

beforeEach(() => {
jest.clearAllMocks();
Expand Down Expand Up @@ -103,8 +103,8 @@ describe('Function: getAppConfig', () => {
'AYADeNgfsRxdKiJ37A12OZ9vN2cAXwABABVhd3MtY3J5cHRvLXB1YmxpYy1rZXkAREF1RzlLMTg1Tkx2Wjk4OGV2UXkyQ1';
const mockNextToken =
'ImRmyljpZnxt7FfxeEOE5H8xQF1SfOlWZFnHujbzJmIvNeSAAA8/qA9ivK0ElRMwpvx96damGxt125XtMkmYf6a0OWSqnBw==';
const mockData = encoder.encode('myAppConfiguration');
const decodedData = decoder.decode(mockData);
const expectedValue = 'my-value';
const mockData = encoder.encode(toBase64(encoder.encode(expectedValue)));

client
.on(StartConfigurationSessionCommand)
Expand All @@ -121,6 +121,6 @@ describe('Function: getAppConfig', () => {
const result = await getAppConfig(name, options);

// Assess
expect(result).toBe(decodedData);
expect(result).toBe(expectedValue);
});
});