Skip to content

Commit fa7deb7

Browse files
dreamorosidreamorosi
committed
wip: other ssmprovider features
1 parent e47ddf6 commit fa7deb7

File tree

3 files changed

+324
-15
lines changed

3 files changed

+324
-15
lines changed

Diff for: ‎packages/parameters/src/BaseProvider.ts

+11-11
Original file line numberDiff line numberDiff line change
@@ -101,30 +101,30 @@ abstract class BaseProvider implements BaseProviderInterface {
101101
return values;
102102
}
103103

104-
/**
105-
* Retrieve parameter value from the underlying parameter store
106-
*
107-
* @param {string} name - Parameter name
108-
* @param {unknown} sdkOptions - Options to pass to the underlying AWS SDK
109-
*/
110-
protected abstract _get(name: string, sdkOptions?: unknown): Promise<string | undefined>;
111-
112-
protected abstract _getMultiple(path: string, sdkOptions?: unknown): Promise<Record<string, string|undefined>>;
113-
114104
/**
115105
* Check whether a key has expired in the cache or not
116106
*
117107
* It returns true if the key is expired or not present in the cache.
118108
*
119109
* @param {string} key - Stringified representation of the key to retrieve
120110
*/
121-
private hasKeyExpiredInCache(key: string): boolean {
111+
public hasKeyExpiredInCache(key: string): boolean {
122112
const value = this.store.get(key);
123113
if (value) return value.isExpired();
124114

125115
return true;
126116
}
127117

118+
/**
119+
* Retrieve parameter value from the underlying parameter store
120+
*
121+
* @param {string} name - Parameter name
122+
* @param {unknown} sdkOptions - Options to pass to the underlying AWS SDK
123+
*/
124+
protected abstract _get(name: string, sdkOptions?: unknown): Promise<string | undefined>;
125+
126+
protected abstract _getMultiple(path: string, sdkOptions?: unknown): Promise<Record<string, string|undefined>>;
127+
128128
}
129129

130130
// TODO: revisit `value` type once we are clearer on the types returned by the various SDKs

Diff for: ‎packages/parameters/src/SSMProvider.ts

+286-4
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,100 @@
1-
import { BaseProvider, DEFAULT_PROVIDERS } from './BaseProvider';
2-
import { SSMClient, GetParameterCommand, paginateGetParametersByPath } from '@aws-sdk/client-ssm';
3-
import type { SSMClientConfig, GetParameterCommandInput, GetParametersByPathCommandInput } from '@aws-sdk/client-ssm';
4-
import type { SSMGetMultipleOptionsInterface, SSMGetOptionsInterface } from 'types/SSMProvider';
1+
import { BaseProvider, DEFAULT_PROVIDERS, transformValue } from './BaseProvider';
2+
import { GetParameterError } from 'Exceptions';
3+
import { DEFAULT_MAX_AGE_SECS } from './constants';
4+
import { SSMClient, GetParameterCommand, paginateGetParametersByPath, GetParametersCommand } from '@aws-sdk/client-ssm';
5+
import type {
6+
SSMClientConfig,
7+
GetParameterCommandInput,
8+
GetParametersByPathCommandInput,
9+
GetParametersCommandInput,
10+
GetParametersCommandOutput,
11+
} from '@aws-sdk/client-ssm';
12+
import type {
13+
SSMGetMultipleOptionsInterface,
14+
SSMGetOptionsInterface,
15+
SSMGetParametersByNameOutputInterface,
16+
SSMGetParametersByNameOptionsInterface,
17+
SSMSplitBatchAndDecryptParametersOutputType,
18+
SSMGetParametersByNameFromCacheOutputType,
19+
} from 'types/SSMProvider';
520
import type { PaginationConfiguration } from '@aws-sdk/types';
621

722
class SSMProvider extends BaseProvider {
823
public client: SSMClient;
24+
protected errorsKey = '_errors';
25+
protected maxGetParametersItems = 10;
926

1027
public constructor(config: SSMClientConfig = {}) {
1128
super();
1229
this.client = new SSMClient(config);
1330
}
1431

32+
/**
33+
* Retrieve multiple parameter values by name from SSM or cache.
34+
*
35+
* `ThrowOnError` decides whether to throw an error if a parameter is not found:
36+
* - A) Default fail-fast behavior: Throws a `GetParameterError` error upon any failure.
37+
* - B) Gracefully aggregate all parameters that failed under "_errors" key.
38+
*
39+
* It transparently uses GetParameter and/or GetParameters depending on decryption requirements.
40+
*
41+
* ```sh
42+
* ┌────────────────────────┐
43+
* ┌───▶ Decrypt entire batch │─────┐
44+
* │ └────────────────────────┘ │ ┌────────────────────┐
45+
* │ ├─────▶ GetParameters API │
46+
* ┌──────────────────┐ │ ┌────────────────────────┐ │ └────────────────────┘
47+
* │ Split batch │─── ┼──▶│ No decryption required │─────┘
48+
* └──────────────────┘ │ └────────────────────────┘
49+
* │ ┌────────────────────┐
50+
* │ ┌────────────────────────┐ │ GetParameter API │
51+
* └──▶│Decrypt some but not all│───────────▶────────────────────┤
52+
* └────────────────────────┘ │ GetParameters API │
53+
* └────────────────────┘
54+
* ```
55+
*
56+
* @param {Record<string, unknown>[]} parameters - List of parameter names, and any optional overrides
57+
*
58+
*/
59+
public async getParametersByName(parameters: Record<string, SSMGetParametersByNameOptionsInterface>, options: SSMGetParametersByNameOptionsInterface): Promise<Record<string, unknown>> {
60+
const configs = { ...{
61+
decrypt: false,
62+
maxAge: DEFAULT_MAX_AGE_SECS,
63+
throwOnError: true,
64+
}, ...options };
65+
66+
let response: Record<string, unknown> = {};
67+
68+
// NOTE: We fail early to avoid unintended graceful errors being replaced with their '_errors' param values
69+
SSMProvider.throwIfErrorsKeyIsPresent(parameters, this.errorsKey, configs.throwOnError);
70+
71+
const { batch, decrypt } = SSMProvider.splitBatchAndDecryptParameters(parameters, configs);
72+
// NOTE: We need to find out whether all parameters must be decrypted or not to know which API to use
73+
// Logic:
74+
// GetParameters API -> When decrypt is used for all parameters in the the batch
75+
// GetParameter API -> When decrypt is used for one or more in the batch
76+
if (Object.keys(decrypt).length !== Object.keys(parameters).length) {
77+
const { response: decryptResponse, errors: decryptErrors } = await this.getParametersByNameWithDecryptOption(decrypt, configs.throwOnError);
78+
const { response: batchResponse, errors: batchErrors } = await this.getParametersBatchByName(batch, configs.throwOnError, false);
79+
80+
response = { ...decryptResponse, ...batchResponse };
81+
// Fail-fast disabled, let's aggregate errors under "_errors" key so they can handle gracefully
82+
if (!configs.throwOnError) {
83+
response[this.errorsKey] = [ ...decryptErrors, ...batchErrors ];
84+
}
85+
} else {
86+
const { response: batchResponse, errors: batchErrors } = await this.getParametersBatchByName(decrypt, configs.throwOnError, true);
87+
88+
response = batchResponse;
89+
// Fail-fast disabled, let's aggregate errors under "_errors" key so they can handle gracefully
90+
if (!configs.throwOnError) {
91+
response[this.errorsKey] = [...batchErrors];
92+
}
93+
}
94+
95+
return response;
96+
}
97+
1598
protected async _get(name: string, options?: SSMGetOptionsInterface): Promise<string | undefined> {
1699
const sdkOptions: GetParameterCommandInput = {
17100
Name: name,
@@ -68,6 +151,205 @@ class SSMProvider extends BaseProvider {
68151

69152
return parameters;
70153
}
154+
155+
protected async _getParametersByName(parameters: Record<string, SSMGetParametersByNameOptionsInterface>, throwOnError: boolean, decrypt: boolean): Promise<SSMGetParametersByNameOutputInterface> {
156+
const results: SSMGetParametersByNameOutputInterface = {
157+
response: {},
158+
errors: [],
159+
};
160+
161+
const sdkOptions: GetParametersCommandInput = {
162+
Names: Object.keys(parameters),
163+
};
164+
if (decrypt) {
165+
sdkOptions.WithDecryption = true;
166+
}
167+
168+
const result = await this.client.send(new GetParametersCommand(sdkOptions));
169+
results.errors = SSMProvider.handleAnyInvalidGetParameterErrors(result, throwOnError);
170+
results.response = this.transformAndCacheGetParametersResponse(result, parameters, throwOnError);
171+
172+
return results;
173+
}
174+
175+
/**
176+
* Slice batch and fetch parameters using GetPrameters API by max permissible batch size
177+
*/
178+
protected async getParametersBatchByName(parameters: Record<string, SSMGetParametersByNameOptionsInterface>, throwOnError: boolean, decrypt: boolean): Promise<SSMGetParametersByNameOutputInterface> {
179+
const results: SSMGetParametersByNameOutputInterface = {
180+
response: {},
181+
errors: [],
182+
};
183+
184+
// Fetch each possible batch param from cache and return if entire batch is cached
185+
const { cached, toFetch } = await this.getParametersByNameFromCache(parameters);
186+
if (Object.keys(cached).length > Object.keys(parameters).length) {
187+
results.response = cached;
188+
189+
return results;
190+
}
191+
192+
// Slice batch by max permitted GetParameters call and retrieve the ones that are not cached
193+
const { response: batchResponse, errors: batchErrors } = await this.getParametersByNameInChunks(toFetch, throwOnError, decrypt);
194+
results.response = { ...cached, ...batchResponse };
195+
results.errors = batchErrors;
196+
197+
return results;
198+
}
199+
200+
/**
201+
* Fetch each parameter from batch that hasn't expired from cache
202+
*/
203+
protected async getParametersByNameFromCache(parameters: Record<string, SSMGetParametersByNameOptionsInterface>): Promise<SSMGetParametersByNameFromCacheOutputType> {
204+
const results: SSMGetParametersByNameFromCacheOutputType = {
205+
cached: {},
206+
toFetch: {},
207+
};
208+
209+
for (const [ parameterName, parameterOptions ] of Object.entries(parameters)) {
210+
const cacheKey = [ parameterName, parameterOptions.transform ].toString();
211+
if (!super.hasKeyExpiredInCache(cacheKey)) {
212+
results.cached[parameterName] = super.store.get(cacheKey);
213+
} else {
214+
results.toFetch[parameterName] = parameterOptions;
215+
}
216+
}
217+
218+
return results;
219+
}
220+
221+
protected async getParametersByNameInChunks(parameters: Record<string, SSMGetParametersByNameOptionsInterface>, throwOnError: boolean, decrypt: boolean): Promise<SSMGetParametersByNameOutputInterface> {
222+
const results: SSMGetParametersByNameOutputInterface = {
223+
response: {},
224+
errors: [],
225+
};
226+
227+
// Slice object into chunks of max permissible batch size
228+
const chunks = Object.entries(parameters).reduce((acc, [ parameterName, parameterOptions ], index) => {
229+
const chunkIndex = Math.floor(index / this.maxGetParametersItems);
230+
if (!acc[chunkIndex]) {
231+
acc[chunkIndex] = {};
232+
}
233+
acc[chunkIndex][parameterName] = parameterOptions;
234+
235+
return acc;
236+
}, [] as Record<string, SSMGetParametersByNameOptionsInterface>[]);
237+
238+
// Fetch each chunk and merge results
239+
for (const chunk of chunks) {
240+
const { response: chunkResponse, errors: chunkErrors } = await this._getParametersByName(chunk, throwOnError, decrypt);
241+
results.response = { ...results.response, ...chunkResponse };
242+
results.errors = [ ...results.errors, ...chunkErrors ];
243+
}
244+
245+
return results;
246+
}
247+
248+
protected async getParametersByNameWithDecryptOption(parameters: Record<string, SSMGetParametersByNameOptionsInterface>, throwOnError: boolean): Promise<SSMGetParametersByNameOutputInterface> {
249+
const results: SSMGetParametersByNameOutputInterface = {
250+
response: {},
251+
errors: [],
252+
};
253+
254+
for (const [ parameterName, parameterOptions ] of Object.entries(parameters)) {
255+
try {
256+
results.response[parameterName] = await this._get(parameterName, parameterOptions);
257+
} catch (error) {
258+
if (throwOnError) {
259+
throw error;
260+
}
261+
results.errors.push(parameterName);
262+
}
263+
}
264+
265+
return results;
266+
}
267+
268+
/**
269+
* Handle any invalid parameters returned by GetParameters API
270+
* GetParameters is non-atomic. Failures don't always reflect in exceptions so we need to collect.
271+
*/
272+
protected static handleAnyInvalidGetParameterErrors(result: GetParametersCommandOutput, throwOnError: boolean): string[] {
273+
const errors: string[] = [];
274+
if (result.InvalidParameters && result.InvalidParameters.length > 0) {
275+
if (throwOnError) {
276+
throw new GetParameterError(`Failed to fetch parameters: ${result.InvalidParameters.join(', ')}`);
277+
}
278+
errors.push(...result.InvalidParameters);
279+
}
280+
281+
return errors;
282+
}
283+
284+
/**
285+
* Split parameters that can be fetched by GetParameters vs GetParameter.
286+
*/
287+
protected static splitBatchAndDecryptParameters(parameters: Record<string, SSMGetParametersByNameOptionsInterface>, configs: SSMGetParametersByNameOptionsInterface): SSMSplitBatchAndDecryptParametersOutputType {
288+
const splitParameters: SSMSplitBatchAndDecryptParametersOutputType = {
289+
batch: {},
290+
decrypt: {},
291+
};
292+
293+
for (const [ parameterName, parameterOptions ] of Object.entries(parameters)) {
294+
const overrides = parameterOptions || {};
295+
overrides.transform = overrides.transform || configs.transform;
296+
297+
if (!overrides.hasOwnProperty('decrypt')) {
298+
overrides.decrypt = configs.decrypt;
299+
}
300+
if (!overrides.hasOwnProperty('maxAge')) {
301+
overrides.maxAge = configs.maxAge;
302+
}
303+
304+
if (overrides.decrypt) {
305+
splitParameters.decrypt[parameterName] = overrides;
306+
} else {
307+
splitParameters.batch[parameterName] = overrides;
308+
}
309+
}
310+
311+
return splitParameters;
312+
}
313+
314+
/**
315+
* Throw a GetParameterError if fail-fast is disabled and `_errors` key is in parameters list.
316+
*
317+
* @param {Record<string, unknown>} parameters
318+
* @param {string} reservedParameter
319+
* @param {boolean} throwOnError
320+
*/
321+
protected static throwIfErrorsKeyIsPresent(parameters: Record<string, unknown>, reservedParameter: string, throwOnError: boolean): void {
322+
if (!throwOnError && parameters.hasOwnProperty(reservedParameter)) {
323+
throw new GetParameterError(`You cannot fetch a parameter named ${reservedParameter} in graceful error mode.`);
324+
}
325+
}
326+
327+
protected transformAndCacheGetParametersResponse(response: GetParametersCommandOutput, parameters: Record<string, SSMGetParametersByNameOptionsInterface>, throwOnError: boolean): Record<string, unknown> {
328+
const results: Record<string, unknown> = {};
329+
330+
for (const parameter of response.Parameters || []) {
331+
// If the parameter is present in the response, then it has a Name
332+
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
333+
const parameterName = parameter.Name!;
334+
const parameterValue = parameter.Value;
335+
const parameterOptions = parameters[parameterName];
336+
337+
let value;
338+
// NOTE: if transform is set, we do it before caching to reduce number of operations
339+
if (parameterValue && parameterOptions.transform) {
340+
value = transformValue(parameterValue, parameterOptions.transform, throwOnError);
341+
}
342+
343+
if (value) {
344+
const cacheKey = [ parameterName, parameterOptions.transform ].toString();
345+
super.addToCache(cacheKey, value, parameterOptions.maxAge || DEFAULT_MAX_AGE_SECS);
346+
}
347+
348+
results[parameterName] = value;
349+
}
350+
351+
return results;
352+
}
71353
}
72354

73355
const getParameter = (name: string, options?: SSMGetOptionsInterface): Promise<undefined | string | Record<string, unknown>> => {

Diff for: ‎packages/parameters/src/types/SSMProvider.ts

+27
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import type { GetParameterCommandInput, GetParametersByPathCommandInput } from '@aws-sdk/client-ssm';
2+
import { ExpirableValue } from 'BaseProvider';
23
import type { TransformOptions } from 'types/BaseProvider';
34

45
/**
@@ -27,7 +28,33 @@ interface SSMGetMultipleOptionsInterface {
2728
throwOnTransformError?: boolean
2829
}
2930

31+
interface SSMGetParametersByNameOptionsInterface {
32+
maxAge?: number
33+
throwOnError?: boolean
34+
decrypt?: boolean
35+
transform?: TransformOptions
36+
}
37+
38+
type SSMSplitBatchAndDecryptParametersOutputType = {
39+
batch: Record<string, SSMGetParametersByNameOptionsInterface>
40+
decrypt: Record<string, SSMGetParametersByNameOptionsInterface>
41+
} & { [key: string]: SSMGetParametersByNameOptionsInterface };
42+
43+
interface SSMGetParametersByNameOutputInterface {
44+
response: Record<string, unknown>
45+
errors: string[]
46+
}
47+
48+
type SSMGetParametersByNameFromCacheOutputType = {
49+
cached: Record<string, ExpirableValue | undefined>
50+
toFetch: Record<string, ExpirableValue | undefined>
51+
} & { [key: string]: SSMGetParametersByNameOptionsInterface };
52+
3053
export {
3154
SSMGetOptionsInterface,
3255
SSMGetMultipleOptionsInterface,
56+
SSMGetParametersByNameOptionsInterface,
57+
SSMSplitBatchAndDecryptParametersOutputType,
58+
SSMGetParametersByNameOutputInterface,
59+
SSMGetParametersByNameFromCacheOutputType,
3360
};

0 commit comments

Comments
 (0)