Skip to content

Commit 6a4c8fc

Browse files
tsctxtsctx
authored
fix(benchmark): make it fair (#2929)
* fix(benchmark): make it fair * chore: update benchmark results
1 parent 7298524 commit 6a4c8fc

File tree

5 files changed

+153
-152
lines changed

5 files changed

+153
-152
lines changed

‎benchmarks/_util/index.js

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
'use strict'
2+
3+
const parallelRequests = parseInt(process.env.PARALLEL, 10) || 100
4+
5+
function makeParallelRequests (cb) {
6+
const promises = new Array(parallelRequests)
7+
for (let i = 0; i < parallelRequests; ++i) {
8+
promises[i] = new Promise(cb)
9+
}
10+
return Promise.all(promises)
11+
}
12+
13+
function printResults (results) {
14+
// Sort results by least performant first, then compare relative performances and also printing padding
15+
let last
16+
17+
const rows = Object.entries(results)
18+
// If any failed, put on the top of the list, otherwise order by mean, ascending
19+
.sort((a, b) => (!a[1].success ? -1 : b[1].mean - a[1].mean))
20+
.map(([name, result]) => {
21+
if (!result.success) {
22+
return {
23+
Tests: name,
24+
Samples: result.size,
25+
Result: 'Errored',
26+
Tolerance: 'N/A',
27+
'Difference with Slowest': 'N/A'
28+
}
29+
}
30+
31+
// Calculate throughput and relative performance
32+
const { size, mean, standardError } = result
33+
const relative = last !== 0 ? (last / mean - 1) * 100 : 0
34+
35+
// Save the slowest for relative comparison
36+
if (typeof last === 'undefined') {
37+
last = mean
38+
}
39+
40+
return {
41+
Tests: name,
42+
Samples: size,
43+
Result: `${((parallelRequests * 1e9) / mean).toFixed(2)} req/sec`,
44+
Tolerance: `± ${((standardError / mean) * 100).toFixed(2)} %`,
45+
'Difference with slowest':
46+
relative > 0 ? `+ ${relative.toFixed(2)} %` : '-'
47+
}
48+
})
49+
50+
return console.table(rows)
51+
}
52+
53+
module.exports = { makeParallelRequests, printResults }

‎benchmarks/benchmark.js

Lines changed: 37 additions & 67 deletions
Original file line numberDiff line numberDiff line change
@@ -8,20 +8,20 @@ const { isMainThread } = require('node:worker_threads')
88

99
const { Pool, Client, fetch, Agent, setGlobalDispatcher } = require('..')
1010

11+
const { makeParallelRequests, printResults } = require('./_util')
12+
1113
let nodeFetch
1214
const axios = require('axios')
1315
let superagent
1416
let got
1517

16-
const util = require('node:util')
17-
const _request = require('request')
18-
const request = util.promisify(_request)
18+
const { promisify } = require('node:util')
19+
const request = promisify(require('request'))
1920

2021
const iterations = (parseInt(process.env.SAMPLES, 10) || 10) + 1
2122
const errorThreshold = parseInt(process.env.ERROR_THRESHOLD, 10) || 3
2223
const connections = parseInt(process.env.CONNECTIONS, 10) || 50
2324
const pipelining = parseInt(process.env.PIPELINING, 10) || 10
24-
const parallelRequests = parseInt(process.env.PARALLEL, 10) || 100
2525
const headersTimeout = parseInt(process.env.HEADERS_TIMEOUT, 10) || 0
2626
const bodyTimeout = parseInt(process.env.BODY_TIMEOUT, 10) || 0
2727
const dest = {}
@@ -136,53 +136,6 @@ class SimpleRequest {
136136
}
137137
}
138138

139-
function makeParallelRequests (cb) {
140-
const promises = new Array(parallelRequests)
141-
for (let i = 0; i < parallelRequests; ++i) {
142-
promises[i] = new Promise(cb)
143-
}
144-
return Promise.all(promises)
145-
}
146-
147-
function printResults (results) {
148-
// Sort results by least performant first, then compare relative performances and also printing padding
149-
let last
150-
151-
const rows = Object.entries(results)
152-
// If any failed, put on the top of the list, otherwise order by mean, ascending
153-
.sort((a, b) => (!a[1].success ? -1 : b[1].mean - a[1].mean))
154-
.map(([name, result]) => {
155-
if (!result.success) {
156-
return {
157-
Tests: name,
158-
Samples: result.size,
159-
Result: 'Errored',
160-
Tolerance: 'N/A',
161-
'Difference with Slowest': 'N/A'
162-
}
163-
}
164-
165-
// Calculate throughput and relative performance
166-
const { size, mean, standardError } = result
167-
const relative = last !== 0 ? (last / mean - 1) * 100 : 0
168-
169-
// Save the slowest for relative comparison
170-
if (typeof last === 'undefined') {
171-
last = mean
172-
}
173-
174-
return {
175-
Tests: name,
176-
Samples: size,
177-
Result: `${((parallelRequests * 1e9) / mean).toFixed(2)} req/sec`,
178-
Tolerance: `± ${((standardError / mean) * 100).toFixed(2)} %`,
179-
'Difference with slowest': relative > 0 ? `+ ${relative.toFixed(2)} %` : '-'
180-
}
181-
})
182-
183-
return console.table(rows)
184-
}
185-
186139
const experiments = {
187140
'http - no keepalive' () {
188141
return makeParallelRequests(resolve => {
@@ -217,8 +170,8 @@ const experiments = {
217170
'undici - pipeline' () {
218171
return makeParallelRequests(resolve => {
219172
dispatcher
220-
.pipeline(undiciOptions, data => {
221-
return data.body
173+
.pipeline(undiciOptions, ({ body }) => {
174+
return body
222175
})
223176
.end()
224177
.pipe(
@@ -288,9 +241,15 @@ if (process.env.PORT) {
288241
})
289242
}
290243

244+
const axiosOptions = {
245+
url: dest.url,
246+
method: 'GET',
247+
responseType: 'stream',
248+
httpAgent: axiosAgent
249+
}
291250
experiments.axios = () => {
292251
return makeParallelRequests(resolve => {
293-
axios.get(dest.url, { responseType: 'stream', httpAgent: axiosAgent }).then(res => {
252+
axios.request(axiosOptions).then(res => {
294253
res.data.pipe(new Writable({
295254
write (chunk, encoding, callback) {
296255
callback()
@@ -300,26 +259,37 @@ if (process.env.PORT) {
300259
})
301260
}
302261

262+
const gotOptions = {
263+
url: dest.url,
264+
method: 'GET',
265+
agent: {
266+
http: gotAgent
267+
},
268+
// avoid body processing
269+
isStream: true
270+
}
303271
experiments.got = () => {
304272
return makeParallelRequests(resolve => {
305-
got.get(dest.url, { agent: { http: gotAgent } }).then(res => {
306-
res.pipe(new Writable({
307-
write (chunk, encoding, callback) {
308-
callback()
309-
}
310-
})).on('finish', resolve)
311-
}).catch(console.log)
273+
got(gotOptions).pipe(new Writable({
274+
write (chunk, encoding, callback) {
275+
callback()
276+
}
277+
})).on('finish', resolve)
312278
})
313279
}
314280

281+
const requestOptions = {
282+
url: dest.url,
283+
method: 'GET',
284+
agent: requestAgent,
285+
// avoid body toString
286+
encoding: null
287+
}
315288
experiments.request = () => {
316289
return makeParallelRequests(resolve => {
317-
request(dest.url, { agent: requestAgent }).then(res => {
318-
res.pipe(new Writable({
319-
write (chunk, encoding, callback) {
320-
callback()
321-
}
322-
})).on('finish', resolve)
290+
request(requestOptions).then(() => {
291+
// already body consumed
292+
resolve()
323293
}).catch(console.log)
324294
})
325295
}

‎benchmarks/post-benchmark.js

Lines changed: 21 additions & 67 deletions
Original file line numberDiff line numberDiff line change
@@ -5,22 +5,23 @@ const os = require('node:os')
55
const path = require('node:path')
66
const { Writable, Readable, pipeline } = require('node:stream')
77
const { isMainThread } = require('node:worker_threads')
8+
89
const { Pool, Client, fetch, Agent, setGlobalDispatcher } = require('..')
910

11+
const { makeParallelRequests, printResults } = require('./_util')
12+
1013
let nodeFetch
1114
const axios = require('axios')
1215
let superagent
1316
let got
1417

15-
const util = require('node:util')
16-
const _request = require('request')
17-
const request = util.promisify(_request)
18+
const { promisify } = require('node:util')
19+
const request = promisify(require('request'))
1820

1921
const iterations = (parseInt(process.env.SAMPLES, 10) || 10) + 1
2022
const errorThreshold = parseInt(process.env.ERROR_THRESHOLD, 10) || 3
2123
const connections = parseInt(process.env.CONNECTIONS, 10) || 50
2224
const pipelining = parseInt(process.env.PIPELINING, 10) || 10
23-
const parallelRequests = parseInt(process.env.PARALLEL, 10) || 100
2425
const headersTimeout = parseInt(process.env.HEADERS_TIMEOUT, 10) || 0
2526
const bodyTimeout = parseInt(process.env.BODY_TIMEOUT, 10) || 0
2627
const dest = {}
@@ -147,53 +148,6 @@ class SimpleRequest {
147148
}
148149
}
149150

150-
function makeParallelRequests (cb) {
151-
const promises = new Array(parallelRequests)
152-
for (let i = 0; i < parallelRequests; ++i) {
153-
promises[i] = new Promise(cb)
154-
}
155-
return Promise.all(promises)
156-
}
157-
158-
function printResults (results) {
159-
// Sort results by least performant first, then compare relative performances and also printing padding
160-
let last
161-
162-
const rows = Object.entries(results)
163-
// If any failed, put on the top of the list, otherwise order by mean, ascending
164-
.sort((a, b) => (!a[1].success ? -1 : b[1].mean - a[1].mean))
165-
.map(([name, result]) => {
166-
if (!result.success) {
167-
return {
168-
Tests: name,
169-
Samples: result.size,
170-
Result: 'Errored',
171-
Tolerance: 'N/A',
172-
'Difference with Slowest': 'N/A'
173-
}
174-
}
175-
176-
// Calculate throughput and relative performance
177-
const { size, mean, standardError } = result
178-
const relative = last !== 0 ? (last / mean - 1) * 100 : 0
179-
180-
// Save the slowest for relative comparison
181-
if (typeof last === 'undefined') {
182-
last = mean
183-
}
184-
185-
return {
186-
Tests: name,
187-
Samples: size,
188-
Result: `${((parallelRequests * 1e9) / mean).toFixed(2)} req/sec`,
189-
Tolerance: `± ${((standardError / mean) * 100).toFixed(2)} %`,
190-
'Difference with slowest': relative > 0 ? `+ ${relative.toFixed(2)} %` : '-'
191-
}
192-
})
193-
194-
return console.table(rows)
195-
}
196-
197151
const experiments = {
198152
'http - no keepalive' () {
199153
return makeParallelRequests(resolve => {
@@ -236,8 +190,8 @@ const experiments = {
236190
this.push(null)
237191
}
238192
}),
239-
dispatcher.pipeline(undiciOptions, data => {
240-
return data.body
193+
dispatcher.pipeline(undiciOptions, ({ body }) => {
194+
return body
241195
}),
242196
new Writable({
243197
write (chunk, encoding, callback) {
@@ -341,22 +295,23 @@ if (process.env.PORT) {
341295
}
342296

343297
const gotOptions = {
298+
url: dest.url,
344299
method: 'POST',
345300
headers,
346301
agent: {
347302
http: gotAgent
348303
},
304+
// avoid body processing
305+
isStream: true,
349306
body: data
350307
}
351308
experiments.got = () => {
352309
return makeParallelRequests(resolve => {
353-
got(dest.url, gotOptions).then(res => {
354-
res.pipe(new Writable({
355-
write (chunk, encoding, callback) {
356-
callback()
357-
}
358-
})).on('finish', resolve)
359-
}).catch(console.log)
310+
got(gotOptions).pipe(new Writable({
311+
write (chunk, encoding, callback) {
312+
callback()
313+
}
314+
})).on('finish', resolve)
360315
})
361316
}
362317

@@ -365,16 +320,15 @@ if (process.env.PORT) {
365320
method: 'POST',
366321
headers,
367322
agent: requestAgent,
368-
body: data
323+
body: data,
324+
// avoid body toString
325+
encoding: null
369326
}
370327
experiments.request = () => {
371328
return makeParallelRequests(resolve => {
372-
request(requestOptions).then(res => {
373-
res.pipe(new Writable({
374-
write (chunk, encoding, callback) {
375-
callback()
376-
}
377-
})).on('finish', resolve)
329+
request(requestOptions).then(() => {
330+
// already body consumed
331+
resolve()
378332
}).catch(console.log)
379333
})
380334
}

‎benchmarks/server.js

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,10 +24,16 @@ if (cluster.isPrimary) {
2424
}
2525
} else {
2626
const buf = Buffer.alloc(64 * 1024, '_')
27+
28+
const headers = {
29+
'Content-Length': `${buf.byteLength}`,
30+
'Content-Type': 'text/plain; charset=UTF-8'
31+
}
2732
let i = 0
28-
const server = createServer((req, res) => {
33+
const server = createServer((_req, res) => {
2934
i++
30-
setTimeout(function () {
35+
setTimeout(() => {
36+
res.writeHead(200, headers)
3137
res.end(buf)
3238
}, timeout)
3339
}).listen(port)

0 commit comments

Comments
 (0)