Skip to content

Commit d95add3

Browse files
authored
fix(cli): trace output (-vv) is useless when files are uploaded (#33104)
### Reason for this change When running the CDK CLI in trace mode (with `-vv`), e.g. `cdk deploy -vv` we log all SDK calls and their inputs. For file uploads, the input contains the buffer of the file to be uploaded. This results in really bad output: ``` [20:14:31] [sdk info] S3.PutObject({"Bucket":"cdk-hnb659fds-assets-us-east-1","Key":"220c435f80d2dd2cfc5dd0e8e29bfbb6ec58892bd813f5d6a008fdda38f6c02f.zip","Body":{"type":"Buffer","data":[80,75,3,4,20,0,8,0,8,0,0,0,33,0,0,0,0,0,0,0,0,0,0,0,0,0,8,0,0,0,100,97,116,97,46,116,120,116,75,73,44,73,52,227,2,0,80,75,7,8,165,92,116,66,8,0,0,0,6,0,0,0,80,75,1,2,45,3,20,0,8,0,8,0,0,0,33,0,165,92,116,66,8,0,0,0,6,0,0,0,8,0,0,0,0,0,0,0,0,0,32,0,164,129,0,0,0,0,100,97,116,97,46,116,120,116,80,75,5,6,0,0,0,0,1,0,1,0,54,0,0,0,62,0,0,0,0,0]},"ContentType":"application/zip","ChecksumAlgorithm":"SHA256","ServerSideEncryption":"aws:kms"}) -> OK ``` Now in this case we are only uploading 138 bytes. Imagine what this log looks like for files of 10MiB or 100MiB. Reportedly this also crashed every terminal. ### Description of changes There's no need to log the buffer bytes. Instead replace it with a string indicating its a buffer and the buffer length: ``` [20:14:31] [sdk info] S3.PutObject({"Body":"<Buffer 138 Bytes>"}) -> OK ``` ### Describe any new or updated permissions being added n/a ### Description of how you validated changes Tests ### Checklist - [x] My code adheres to the [CONTRIBUTING GUIDE](https://github.com/aws/aws-cdk/blob/main/CONTRIBUTING.md) and [DESIGN GUIDELINES](https://github.com/aws/aws-cdk/blob/main/docs/DESIGN_GUIDELINES.md) ---- *By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license*
1 parent c0a2cbf commit d95add3

File tree

5 files changed

+81
-1
lines changed

5 files changed

+81
-1
lines changed

packages/aws-cdk/lib/api/aws-auth/sdk-logger.ts

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import { inspect } from 'util';
22
import { Logger } from '@smithy/types';
33
import { trace } from '../../logging';
4+
import { replacerBufferWithInfo } from '../../serialize';
45

56
export class SdkToCliLogger implements Logger {
67
public trace(..._content: any[]) {
@@ -105,7 +106,7 @@ function formatApiCall(content: any): string | undefined {
105106
parts.push(`[${content.metadata?.attempts} attempts, ${content.metadata?.totalRetryDelay}ms retry]`);
106107
}
107108

108-
parts.push(`${service}.${api}(${JSON.stringify(content.input)})`);
109+
parts.push(`${service}.${api}(${JSON.stringify(content.input, replacerBufferWithInfo)})`);
109110

110111
if (isSdkApiCallSuccess(content)) {
111112
parts.push('-> OK');

packages/aws-cdk/lib/serialize.ts

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import * as fs from 'fs-extra';
2+
import { formatBytes } from './util/bytes';
23
import * as yaml_cfn from './util/yaml-cfn';
34

45
/**
@@ -51,3 +52,35 @@ export function obscureTemplate(template: any = {}) {
5152

5253
return template;
5354
}
55+
56+
/**
57+
* Detects a buffer that has been converted to a JSON-like object
58+
* In Node, `Buffer`s have `toJSON()` method that converts the buffer
59+
* into a JS object that can be JSON stringified.
60+
* Unfortunately this conversion happens before the replacer is called,
61+
* so normal means of detecting a `Buffer` objet don't work anymore.
62+
* @see https://github.com/nodejs/node-v0.x-archive/issues/5110
63+
*/
64+
function isJsonBuffer(value: any): value is {
65+
type: 'Buffer';
66+
data: number[];
67+
} {
68+
return typeof value === 'object'
69+
&& 'type' in value
70+
&& value.type === 'Buffer'
71+
&& 'data' in value
72+
&& Array.isArray(value.data);
73+
}
74+
75+
/**
76+
* A JSON.stringify() replacer that converts Buffers into a string with information
77+
* Use this if you plan to print out JSON stringified objects that may contain a Buffer.
78+
* Without this, large buffers (think: Megabytes) will completely fill up the output
79+
* and even crash the system.
80+
*/
81+
export function replacerBufferWithInfo(_key: any, value: any): any {
82+
if (isJsonBuffer(value)) {
83+
return `<Buffer: ${formatBytes(value.data.length)}>`;
84+
}
85+
return value;
86+
}

packages/aws-cdk/lib/util/bytes.ts

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
/**
2+
* Format bytes as a human readable string
3+
*
4+
* @param bytes Number of bytes to format
5+
* @param decimals Number of decimal places to show (default 2)
6+
* @returns Formatted string with appropriate unit suffix
7+
*/
8+
export function formatBytes(bytes: number, decimals: number = 2): string {
9+
decimals = decimals < 0 ? 0 : decimals;
10+
11+
if (bytes === 0) {
12+
return '0 Bytes';
13+
}
14+
15+
const k = 1024;
16+
const sizes = ['Bytes', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB', 'YiB'];
17+
18+
const i = Math.floor(Math.log(bytes) / Math.log(k));
19+
20+
return `${parseFloat((bytes / Math.pow(k, i)).toFixed(decimals))} ${sizes[i]}`;
21+
}
Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
import { formatBytes } from '../../lib/util/bytes';
2+
3+
test.each([
4+
[0, '0 Bytes'],
5+
[10, '10 Bytes'],
6+
[1024, '1 KiB'],
7+
[10.5 * 1024 * 1024, '10.5 MiB'],
8+
])('converts %s bytes to %s', (bytes: number, expected: string) => {
9+
expect(formatBytes(bytes)).toEqual(expected);
10+
});
11+
12+
test('can format many decimals', () => {
13+
expect(formatBytes(10.51234 * 1024 * 1024, 5)).toEqual('10.51234 MiB');
14+
});
15+
16+
test('can deal with negative decimals', () => {
17+
expect(formatBytes(10.5 * 1024 * 1024, -1)).toEqual('11 MiB');
18+
});
Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
import { replacerBufferWithInfo } from '../../lib/serialize';
2+
3+
test('converts buffer to information', () => {
4+
const res = JSON.stringify({ data: Buffer.from('test data') }, replacerBufferWithInfo);
5+
6+
expect(res).toEqual('{"data":"<Buffer: 9 Bytes>"}');
7+
});

0 commit comments

Comments
 (0)