Skip to content

Commit 28db6e3

Browse files
RomainMullerRomainMuller
authored
fix(benchmark): compilation fails due to missing dependencies (#3610)
Also, adding a reference benchmark building the same code using the TypeScript compiler only. --- By submitting this pull request, I confirm that my contribution is made under the terms of the [Apache 2.0 license]. [Apache 2.0 license]: https://www.apache.org/licenses/LICENSE-2.0
1 parent 6cb83eb commit 28db6e3

File tree

9 files changed

+672
-75
lines changed

9 files changed

+672
-75
lines changed

‎packages/@jsii/benchmarks/bin/benchmark.ts

+22
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,28 @@ interface ResultsJson {
9696
Promise.resolve([]),
9797
);
9898

99+
// If we are running in GitHub Actions, emit a summary document.
100+
if (process.env.GITHUB_STEP_SUMMARY != null) {
101+
await fs.writeFile(
102+
process.env.GITHUB_STEP_SUMMARY,
103+
[
104+
'## Benchmark Results',
105+
'',
106+
'Suite | Avg | StdDev',
107+
'------|-----|-------',
108+
...resultsJson
109+
.sort((l, r) => l.name.localeCompare(r.name))
110+
.map(
111+
({ name, value, range }) =>
112+
`${name} | ${value.toFixed(1)} | ${Math.sqrt(range).toFixed(
113+
2,
114+
)}`,
115+
),
116+
].join('\n'),
117+
'utf-8',
118+
);
119+
}
120+
99121
if (argv.output) {
100122
await fs.writeJson(argv.output, resultsJson, { spaces: 2 });
101123
console.log(`results written to ${argv.output}`);

‎packages/@jsii/benchmarks/fixtures/[email protected]

32.9 KB
Binary file not shown.

‎packages/@jsii/benchmarks/lib/benchmark.ts

+53-11
Original file line numberDiff line numberDiff line change
@@ -186,10 +186,15 @@ export class Benchmark<C> {
186186
if (this.#profile) {
187187
profiler = await this.startProfiler();
188188
}
189-
wrapped(ctx);
190-
const profile = await this.killProfiler(profiler);
191-
const perf = await observer;
192-
this.#afterEach(ctx);
189+
let profile: Profiler.Profile | undefined;
190+
let perf: PerformanceEntry;
191+
try {
192+
wrapped(ctx);
193+
profile = await this.killProfiler(profiler);
194+
perf = await observer;
195+
} finally {
196+
this.#afterEach(ctx);
197+
}
193198

194199
i++;
195200
yield { profile, performance: perf };
@@ -204,19 +209,52 @@ export class Benchmark<C> {
204209
const iterations: Iteration[] = [];
205210
const c = await this.#setup?.();
206211

212+
const durations = new Array<number>();
213+
let min = Number.POSITIVE_INFINITY;
214+
let max = Number.NEGATIVE_INFINITY;
215+
let sum = 0;
216+
let average = 0;
217+
218+
let id = 0;
219+
const padding = this.#iterations.toString().length;
207220
for await (const result of this.runIterations(c)) {
221+
id += 1;
222+
const duration = result.performance.duration;
223+
durations.push(duration);
224+
if (min > duration) {
225+
min = duration;
226+
}
227+
if (max < duration) {
228+
max = duration;
229+
}
230+
sum += duration;
231+
average = sum / id;
232+
233+
const idStr = id.toString().padStart(padding, ' ');
234+
const durStr = duration.toFixed(0);
235+
const eta = new Date(Date.now() + average * (this.#iterations - id));
236+
const pct = (100 * id) / this.#iterations;
237+
238+
this.log(
239+
`Iteration ${idStr}/${this.#iterations} (${pct.toFixed(
240+
0,
241+
)}%) | Duration: ${durStr}ms | ETA ${eta.toISOString()}`,
242+
);
208243
iterations.push(result);
209244
}
210245

211246
this.#teardown(c);
212247

213-
const durations = iterations.map((i) => i.performance.duration);
214-
const max = Math.max(...durations);
215-
const min = Math.min(...durations);
216-
const variance = max - min;
217-
const average =
218-
durations.reduce((accum, duration) => accum + duration, 0) /
219-
durations.length;
248+
// Variance is the average of the squared differences from the mean
249+
const variance =
250+
durations
251+
.map((duration) => Math.pow(duration - average, 2))
252+
.reduce((accum, squareDev) => accum + squareDev) / durations.length;
253+
254+
// Standard deviation is the square root of variance
255+
const stdDev = Math.sqrt(variance);
256+
257+
this.log(`Completed: ${average.toFixed(0)}±${stdDev.toFixed(0)}ms`);
220258

221259
return {
222260
name: this.name,
@@ -227,4 +265,8 @@ export class Benchmark<C> {
227265
iterations,
228266
};
229267
}
268+
269+
private log(message: string) {
270+
console.log(`${new Date().toISOString()} | ${this.name} | ${message}`);
271+
}
230272
}

‎packages/@jsii/benchmarks/lib/index.ts

+166-37
Original file line numberDiff line numberDiff line change
@@ -4,43 +4,172 @@ import { Compiler } from 'jsii/lib/compiler';
44
import { loadProjectInfo } from 'jsii/lib/project-info';
55
import * as os from 'os';
66
import * as path from 'path';
7+
import * as ts from 'typescript-3.9';
78

89
import { Benchmark } from './benchmark';
910
import { cdkv2_21_1, cdkTagv2_21_1 } from './constants';
10-
import { streamUntar } from './util';
11-
12-
// Always run against the same version of CDK source
13-
const cdk = new Benchmark(`Compile aws-cdk-lib@${cdkTagv2_21_1}`)
14-
.setup(async () => {
15-
const sourceDir = fs.mkdtempSync(
16-
path.join(os.tmpdir(), 'jsii-cdk-bench-snapshot'),
17-
);
18-
await streamUntar(cdkv2_21_1, { cwd: sourceDir });
19-
cp.execSync('npm ci', { cwd: sourceDir });
20-
21-
// Working directory for benchmark
22-
const workingDir = fs.mkdtempSync(
23-
path.join(os.tmpdir(), `jsii-cdk-bench@${cdkTagv2_21_1}`),
24-
);
25-
26-
return {
27-
workingDir,
28-
sourceDir,
29-
} as const;
30-
})
31-
.beforeEach(({ workingDir, sourceDir }) => {
32-
fs.removeSync(workingDir);
33-
fs.copySync(sourceDir, workingDir);
34-
})
35-
.subject(({ workingDir }) => {
36-
const { projectInfo } = loadProjectInfo(workingDir);
37-
const compiler = new Compiler({ projectInfo });
38-
39-
compiler.emit();
40-
})
41-
.teardown(({ workingDir, sourceDir }) => {
42-
fs.removeSync(workingDir);
43-
fs.removeSync(sourceDir);
44-
});
45-
46-
export const benchmarks = [cdk];
11+
import { inDirectory, streamUntar } from './util';
12+
13+
// Using the local `npm` package (from dependencies)
14+
const npm = path.resolve(__dirname, '..', 'node_modules', '.bin', 'npm');
15+
16+
export const benchmarks = [
17+
// Reference comparison using the TypeScript compiler
18+
new Benchmark(`Compile aws-cdk-lib@${cdkTagv2_21_1} (tsc)`)
19+
.setup(async () => {
20+
const sourceDir = fs.mkdtempSync(
21+
path.join(os.tmpdir(), 'jsii-cdk-bench-snapshot'),
22+
);
23+
await streamUntar(cdkv2_21_1, { cwd: sourceDir });
24+
cp.execSync(`${npm} ci`, { cwd: sourceDir });
25+
26+
// Working directory for benchmark
27+
const workingDir = fs.mkdtempSync(
28+
path.join(os.tmpdir(), `tsc-cdk-bench@${cdkTagv2_21_1}`),
29+
);
30+
31+
return {
32+
workingDir,
33+
sourceDir,
34+
} as const;
35+
})
36+
.beforeEach(({ workingDir, sourceDir }) => {
37+
fs.removeSync(workingDir);
38+
fs.copySync(sourceDir, workingDir);
39+
})
40+
.subject(({ workingDir }) =>
41+
inDirectory(workingDir, () => {
42+
const { host, options, rootNames } = (function () {
43+
const parsed = ts.parseJsonConfigFileContent(
44+
fs.readJsonSync(path.join(workingDir, 'tsconfig.json')),
45+
ts.sys,
46+
workingDir,
47+
{
48+
module: ts.ModuleKind.CommonJS,
49+
moduleResolution: ts.ModuleResolutionKind.NodeJs,
50+
newLine: ts.NewLineKind.LineFeed,
51+
tsBuildInfoFile: 'tsconfig.tsbuildinfo',
52+
},
53+
'tsconfig.json',
54+
);
55+
56+
const host = ts.createIncrementalCompilerHost(parsed.options, ts.sys);
57+
58+
return {
59+
host,
60+
options: parsed.options,
61+
rootNames: [
62+
...parsed.fileNames,
63+
...(parsed.options.lib && host.getDefaultLibLocation != null
64+
? parsed.options.lib.map((lib) =>
65+
path.join(host.getDefaultLibLocation!(), lib),
66+
)
67+
: []),
68+
],
69+
};
70+
})();
71+
72+
const program = ts
73+
.createIncrementalProgram({
74+
createProgram: ts.createEmitAndSemanticDiagnosticsBuilderProgram,
75+
host,
76+
options,
77+
rootNames,
78+
})
79+
.getProgram();
80+
81+
const preEmitDiagnostics = ts.getPreEmitDiagnostics(program);
82+
if (
83+
preEmitDiagnostics.some(
84+
(diag) => diag.category === ts.DiagnosticCategory.Error,
85+
)
86+
) {
87+
console.error(
88+
ts.formatDiagnosticsWithColorAndContext(
89+
preEmitDiagnostics
90+
.filter((diag) => diag.category === ts.DiagnosticCategory.Error)
91+
.slice(0, 10),
92+
host,
93+
),
94+
);
95+
throw new Error(`TypeScript compiler emitted pre-emit errors!`);
96+
}
97+
98+
const emitResult = program.emit();
99+
if (
100+
emitResult.diagnostics.some(
101+
(diag) => diag.category === ts.DiagnosticCategory.Error,
102+
)
103+
) {
104+
console.error(
105+
ts.formatDiagnosticsWithColorAndContext(
106+
emitResult.diagnostics.filter(
107+
(diag) => diag.category === ts.DiagnosticCategory.Error,
108+
),
109+
host,
110+
),
111+
);
112+
throw new Error(`TypeScript compiler emitted errors!`);
113+
}
114+
}),
115+
)
116+
.teardown(({ workingDir, sourceDir }) => {
117+
fs.removeSync(workingDir);
118+
fs.removeSync(sourceDir);
119+
}),
120+
121+
// Always run against the same version of CDK source
122+
new Benchmark(`Compile aws-cdk-lib@${cdkTagv2_21_1}`)
123+
.setup(async () => {
124+
const sourceDir = fs.mkdtempSync(
125+
path.join(os.tmpdir(), 'jsii-cdk-bench-snapshot'),
126+
);
127+
await streamUntar(cdkv2_21_1, { cwd: sourceDir });
128+
cp.execSync(`${npm} ci`, { cwd: sourceDir });
129+
130+
// Working directory for benchmark
131+
const workingDir = fs.mkdtempSync(
132+
path.join(os.tmpdir(), `jsii-cdk-bench@${cdkTagv2_21_1}`),
133+
);
134+
135+
return {
136+
workingDir,
137+
sourceDir,
138+
} as const;
139+
})
140+
.beforeEach(({ workingDir, sourceDir }) => {
141+
fs.removeSync(workingDir);
142+
fs.copySync(sourceDir, workingDir);
143+
})
144+
.subject(({ workingDir }) =>
145+
inDirectory(workingDir, () => {
146+
const { projectInfo } = loadProjectInfo(workingDir);
147+
const compiler = new Compiler({ projectInfo });
148+
149+
const result = compiler.emit();
150+
if (
151+
result.diagnostics.some(
152+
(diag) => diag.category === ts.DiagnosticCategory.Error,
153+
)
154+
) {
155+
console.error(
156+
ts.formatDiagnosticsWithColorAndContext(
157+
result.diagnostics.filter(
158+
(diag) => diag.category === ts.DiagnosticCategory.Error,
159+
),
160+
{
161+
getCurrentDirectory: () => workingDir,
162+
getCanonicalFileName: path.resolve,
163+
getNewLine: () => ts.sys.newLine,
164+
},
165+
),
166+
);
167+
throw new Error(`jsii compiler emitted errors!`);
168+
}
169+
}),
170+
)
171+
.teardown(({ workingDir, sourceDir }) => {
172+
fs.removeSync(workingDir);
173+
fs.removeSync(sourceDir);
174+
}),
175+
];

‎packages/@jsii/benchmarks/lib/util.ts

+10
Original file line numberDiff line numberDiff line change
@@ -9,3 +9,13 @@ export async function streamUntar(file: string, config: tar.ExtractOptions) {
99
stream.on('error', (error: Error) => ko(error));
1010
});
1111
}
12+
13+
export function inDirectory<T>(newWorkDir: string, cb: () => T) {
14+
const cwd = process.cwd();
15+
try {
16+
process.chdir(newWorkDir);
17+
return cb();
18+
} finally {
19+
process.chdir(cwd);
20+
}
21+
}

‎packages/@jsii/benchmarks/package.json

+1
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
"dependencies": {
88
"fs-extra": "^10.1.0",
99
"jsii": "^0.0.0",
10+
"npm": "^8.12.1",
1011
"tar": "^6.1.11",
1112
"typescript-3.9": "npm:typescript@~3.9.10",
1213
"yargs": "^16.2.0"

0 commit comments

Comments
 (0)