Skip to content

Commit 7f7f8ce

Browse files
dreamorosiam29d
dreamorosi
and
am29d
authored
fix(parser): DynamoDBStream schema & envelope (#3482)
Co-authored-by: Alexander Schueren <[email protected]>
1 parent 8692de6 commit 7f7f8ce

File tree

10 files changed

+777
-179
lines changed

10 files changed

+777
-179
lines changed

Diff for: ‎packages/commons/package.json

+8
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,10 @@
4949
"import": "./lib/esm/LRUCache.js",
5050
"require": "./lib/cjs/LRUCache.js"
5151
},
52+
"./utils/unmarshallDynamoDB": {
53+
"import": "./lib/esm/unmarshallDynamoDB.js",
54+
"require": "./lib/cjs/unmarshallDynamoDB.js"
55+
},
5256
"./types": {
5357
"import": "./lib/esm/types/index.js",
5458
"require": "./lib/cjs/types/index.js"
@@ -68,6 +72,10 @@
6872
"lib/cjs/LRUCache.d.ts",
6973
"lib/esm/LRUCache.d.ts"
7074
],
75+
"utils/unmarshallDynamoDB": [
76+
"lib/cjs/unmarshallDynamoDB.d.ts",
77+
"lib/esm/unmarshallDynamoDB.d.ts"
78+
],
7179
"types": [
7280
"lib/cjs/types/index.d.ts",
7381
"lib/esm/types/index.d.ts"

Diff for: ‎packages/commons/src/unmarshallDynamoDB.ts

+83
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
import type { AttributeValue } from '@aws-sdk/client-dynamodb';
2+
3+
class UnmarshallDynamoDBAttributeError extends Error {
4+
constructor(message: string) {
5+
super(message);
6+
this.name = 'UnmarshallDynamoDBAttributeError';
7+
}
8+
}
9+
10+
// biome-ignore lint/suspicious/noExplicitAny: we need to use any here to support the different types of DynamoDB attributes
11+
const typeHandlers: Record<string, (value: any) => unknown> = {
12+
NULL: () => null,
13+
S: (value) => value,
14+
B: (value) => value,
15+
BS: (value) => new Set(value),
16+
SS: (value) => new Set(value),
17+
BOOL: (value) => Boolean(value),
18+
N: (value) => convertNumber(value),
19+
NS: (value) => new Set((value as Array<string>).map(convertNumber)),
20+
L: (value) => (value as Array<AttributeValue>).map(convertAttributeValue),
21+
M: (value) =>
22+
Object.entries(value).reduce(
23+
(acc, [key, value]) => {
24+
acc[key] = convertAttributeValue(value as AttributeValue);
25+
return acc;
26+
},
27+
{} as Record<string, unknown>
28+
),
29+
};
30+
31+
const convertAttributeValue = (
32+
data: AttributeValue | Record<string, AttributeValue>
33+
): unknown => {
34+
const [type, value] = Object.entries(data)[0];
35+
36+
if (value !== undefined) {
37+
const handler = typeHandlers[type];
38+
if (!handler) {
39+
throw new UnmarshallDynamoDBAttributeError(
40+
`Unsupported type passed: ${type}`
41+
);
42+
}
43+
44+
return handler(value);
45+
}
46+
47+
throw new UnmarshallDynamoDBAttributeError(
48+
`Value is undefined for type: ${type}`
49+
);
50+
};
51+
52+
const convertNumber = (numString: string) => {
53+
const num = Number(numString);
54+
const infinityValues = [Number.POSITIVE_INFINITY, Number.NEGATIVE_INFINITY];
55+
const isLargeFiniteNumber =
56+
(num > Number.MAX_SAFE_INTEGER || num < Number.MIN_SAFE_INTEGER) &&
57+
!infinityValues.includes(num);
58+
if (isLargeFiniteNumber) {
59+
try {
60+
return BigInt(numString);
61+
} catch (error) {
62+
throw new UnmarshallDynamoDBAttributeError(
63+
`${numString} can't be converted to BigInt`
64+
);
65+
}
66+
}
67+
return num;
68+
};
69+
70+
/**
71+
* Unmarshalls a DynamoDB AttributeValue to a JavaScript object.
72+
*
73+
* The implementation is loosely based on the official AWS SDK v3 unmarshall function but
74+
* without support the customization options and with assumed support for BigInt.
75+
*
76+
* @param data - The DynamoDB AttributeValue to unmarshall
77+
*/
78+
const unmarshallDynamoDB = (
79+
data: AttributeValue | Record<string, AttributeValue>
80+
// @ts-expect-error - We intentionally wrap the data into a Map to allow for nested structures
81+
) => convertAttributeValue({ M: data });
82+
83+
export { unmarshallDynamoDB, UnmarshallDynamoDBAttributeError };

Diff for: ‎packages/commons/tests/unit/unmarshallDynamoDB.test.ts

+246
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,246 @@
1+
import { describe, expect, it } from 'vitest';
2+
import {
3+
UnmarshallDynamoDBAttributeError,
4+
unmarshallDynamoDB,
5+
} from '../../src/unmarshallDynamoDB.js';
6+
7+
describe('Function: unmarshallDynamoDB', () => {
8+
it('unmarshalls a DynamoDB string attribute', () => {
9+
// Prepare
10+
const value = { Message: { S: 'test string' } };
11+
12+
// Act
13+
const result = unmarshallDynamoDB(value);
14+
15+
// Assess
16+
expect(result).toStrictEqual({ Message: 'test string' });
17+
});
18+
19+
it('unmarshalls a DynamoDB number attribute', () => {
20+
// Prepare
21+
const value = { Id: { N: '123' } };
22+
23+
// Act
24+
const result = unmarshallDynamoDB(value);
25+
26+
// Assess
27+
expect(result).toStrictEqual({ Id: 123 });
28+
});
29+
30+
it('unmarshalls a DynamoDB boolean attribute', () => {
31+
// Prepare
32+
const value = { Message: { BOOL: true } };
33+
34+
// Act
35+
const result = unmarshallDynamoDB(value);
36+
37+
// Assess
38+
expect(result).toStrictEqual({ Message: true });
39+
});
40+
41+
it('unmarshalls a DynamoDB null attribute', () => {
42+
// Prepare
43+
const value = { Message: { NULL: true } };
44+
45+
// Act
46+
const result = unmarshallDynamoDB(value);
47+
48+
// Assess
49+
expect(result).toStrictEqual({ Message: null });
50+
});
51+
52+
it('unmarshalls a DynamoDB list attribute', () => {
53+
// Prepare
54+
const value = {
55+
Messages: {
56+
L: [{ S: 'string' }, { N: '123' }, { BOOL: true }, { NULL: true }],
57+
},
58+
};
59+
60+
// Act
61+
const result = unmarshallDynamoDB(value);
62+
63+
// Assess
64+
expect(result).toStrictEqual({ Messages: ['string', 123, true, null] });
65+
});
66+
67+
it('unmarshalls a DynamoDB map attribute', () => {
68+
// Prepare
69+
const value = {
70+
Settings: {
71+
M: {
72+
string: { S: 'test' },
73+
number: { N: '123' },
74+
boolean: { BOOL: true },
75+
null: { NULL: true },
76+
},
77+
},
78+
};
79+
80+
// Act
81+
const result = unmarshallDynamoDB(value);
82+
83+
// Assess
84+
expect(result).toStrictEqual({
85+
Settings: {
86+
string: 'test',
87+
number: 123,
88+
boolean: true,
89+
null: null,
90+
},
91+
});
92+
});
93+
94+
it('unmarshalls a DynamoDB string set attribute', () => {
95+
// Prepare
96+
const value = { Messages: { SS: ['a', 'b', 'c'] } };
97+
98+
// Act
99+
const result = unmarshallDynamoDB(value);
100+
101+
// Assess
102+
expect(result).toStrictEqual({ Messages: new Set(['a', 'b', 'c']) });
103+
});
104+
105+
it('unmarshalls a DynamoDB number set attribute', () => {
106+
// Prepare
107+
const value = { Ids: { NS: ['1', '2', '3'] } };
108+
109+
// Act
110+
const result = unmarshallDynamoDB(value);
111+
112+
// Assess
113+
expect(result).toStrictEqual({ Ids: new Set([1, 2, 3]) });
114+
});
115+
116+
it('unmarshalls nested DynamoDB structures', () => {
117+
// Prepare
118+
const value = {
119+
Messages: {
120+
M: {
121+
nested: {
122+
M: {
123+
list: {
124+
L: [
125+
{ S: 'string' },
126+
{
127+
M: {
128+
key: { S: 'value' },
129+
},
130+
},
131+
],
132+
},
133+
},
134+
},
135+
},
136+
},
137+
};
138+
139+
// Act
140+
const result = unmarshallDynamoDB(value);
141+
142+
// Assess
143+
expect(result).toStrictEqual({
144+
Messages: {
145+
nested: {
146+
list: ['string', { key: 'value' }],
147+
},
148+
},
149+
});
150+
});
151+
152+
it('unmarshalls a DynamoDB binary attribute', () => {
153+
// Prepare
154+
const value = {
155+
Data: {
156+
B: new Uint8Array(
157+
Buffer.from('dGhpcyB0ZXh0IGlzIGJhc2U2NC1lbmNvZGVk', 'base64')
158+
),
159+
},
160+
};
161+
162+
// Act
163+
const result = unmarshallDynamoDB(value);
164+
165+
// Assess
166+
expect(result).toStrictEqual({ Data: expect.any(Uint8Array) });
167+
});
168+
169+
it('unmarshalls a DynamoDB binary set attribute', () => {
170+
// Prepare
171+
const value = {
172+
Data: {
173+
BS: [
174+
new Uint8Array(
175+
Buffer.from('dGhpcyB0ZXh0IGlzIGJhc2U2NC1lbmNvZGVk', 'base64')
176+
),
177+
new Uint8Array(
178+
Buffer.from('dGhpcyB0ZXh0IGlzIGJhc2U2NC1lbmNvZGVk', 'base64')
179+
),
180+
],
181+
},
182+
};
183+
184+
// Act
185+
const result = unmarshallDynamoDB(value);
186+
187+
// Assess
188+
expect(result).toStrictEqual({ Data: expect.any(Set) });
189+
});
190+
191+
it('throws if an unsupported type is passed', () => {
192+
// Prepare
193+
const value = { Message: { NNN: '123' } };
194+
195+
// Act & Assess
196+
// @ts-expect-error - Intentionally invalid value
197+
expect(() => unmarshallDynamoDB(value)).toThrow(
198+
UnmarshallDynamoDBAttributeError
199+
);
200+
});
201+
202+
it('unmarshalls a DynamoDB large number attribute to BigInt', () => {
203+
// Prepare
204+
const value = { Balance: { N: '9007199254740992' } }; // Number.MAX_SAFE_INTEGER + 1
205+
206+
// Act
207+
const result = unmarshallDynamoDB(value);
208+
209+
// Assess
210+
expect(result).toStrictEqual({ Balance: BigInt('9007199254740992') });
211+
});
212+
213+
it('unmarshalls a DynamoDB negative large number attribute to BigInt', () => {
214+
// Prepare
215+
const value = { Balance: { N: '-9007199254740992' } }; // Number.MIN_SAFE_INTEGER - 1
216+
217+
// Act
218+
const result = unmarshallDynamoDB(value);
219+
220+
// Assess
221+
expect(result).toStrictEqual({ Balance: BigInt('-9007199254740992') });
222+
});
223+
224+
it('throws when trying to convert an invalid number string to BigInt', () => {
225+
// Prepare
226+
const value = { Balance: { N: '9007199254740992.5' } }; // Invalid BigInt string (decimals not allowed)
227+
228+
// Act & Assess
229+
expect(() => unmarshallDynamoDB(value)).toThrow(
230+
new UnmarshallDynamoDBAttributeError(
231+
"9007199254740992.5 can't be converted to BigInt"
232+
)
233+
);
234+
});
235+
236+
it('throws when no data is found', () => {
237+
// Prepare
238+
const value = undefined;
239+
240+
// Act & Assess
241+
// @ts-expect-error - Intentionally invalid value
242+
expect(() => unmarshallDynamoDB(value)).toThrow(
243+
UnmarshallDynamoDBAttributeError
244+
);
245+
});
246+
});

0 commit comments

Comments
 (0)