benchmark: add progress indicator to compare.js

* Print the progress bar and the current benchmark to stderr
  when stderr is TTY and stdout is not.
* Allow cli arguments without values via setting.boolArgs
* Add --no-progress option

PR-URL: #10823
Fixes: #8659
Reviewed-By: Andreas Madsen <[email protected]>

Author: joyeecheung

benchmark/_benchmark_progress.js

@@ -0,0 +1,120 @@
+'use strict';
+
+const readline = require('readline');
+
+function pad(input, minLength, fill) {
+  var result = input + '';
+  return fill.repeat(Math.max(0, minLength - result.length)) + result;
+}
+
+function fraction(numerator, denominator) {
+  const fdenominator = denominator + '';
+  const fnumerator = pad(numerator, fdenominator.length, ' ');
+  return `${fnumerator}/${fdenominator}`;
+}
+
+function getTime(diff) {
+  const time = Math.ceil(diff[0] + diff[1] / 1e9);
+  const seconds = pad(time % 60, 2, '0');
+  const minutes = pad(Math.floor(time / 60) % (60 * 60), 2, '0');
+  const hours = pad(Math.floor(time / (60 * 60)), 2, '0');
+  return `${hours}:${minutes}:${seconds}`;
+}
+
+// A run is an item in the job queue: { binary, filename, iter }
+// A config is an item in the subqueue: { binary, filename, iter, configs }
+class BenchmarkProgress {
+  constructor(queue, benchmarks) {
+    this.queue = queue;  // Scheduled runs.
+    this.benchmarks = benchmarks;  // Filenames of scheduled benchmarks.
+    this.completedRuns = 0;  // Number of completed runs.
+    this.scheduledRuns = queue.length;  // Number of scheduled runs.
+    // Time when starting to run benchmarks.
+    this.startTime = process.hrtime();
+    // Number of times each file will be run (roughly).
+    this.runsPerFile = queue.length / benchmarks.length;
+    this.currentFile = '';  // Filename of current benchmark.
+    this.currentFileConfig;  // Configurations for current file
+    // Number of configurations already run for the current file.
+    this.completedConfig = 0;
+    // Total number of configurations for the current file
+    this.scheduledConfig = 0;
+    this.interval = 0;  // result of setInterval for updating the elapsed time
+  }
+
+  startQueue(index) {
+    this.kStartOfQueue = index;
+    this.currentFile = this.queue[index].filename;
+    this.interval = setInterval(() => {
+      if (this.completedRuns === this.scheduledRuns) {
+        clearInterval(this.interval);
+      } else {
+        this.updateProgress();
+      }
+    }, 1000);
+  }
+
+  startSubqueue(data, index) {
+    // This subqueue is generated by a new benchmark
+    if (data.name !== this.currentFile || index === this.kStartOfQueue) {
+      this.currentFile = data.name;
+      this.scheduledConfig = data.queueLength;
+    }
+    this.completedConfig = 0;
+    this.updateProgress();
+  }
+
+  completeConfig(data) {
+    this.completedConfig++;
+    this.updateProgress();
+  }
+
+  completeRun(job) {
+    this.completedRuns++;
+    this.updateProgress();
+  }
+
+  getProgress() {
+    // Get time as soon as possible.
+    const diff = process.hrtime(this.startTime);
+
+    const completedRuns = this.completedRuns;
+    const scheduledRuns = this.scheduledRuns;
+    const finished = completedRuns === scheduledRuns;
+
+    // Calculate numbers for fractions.
+    const runsPerFile = this.runsPerFile;
+    const completedFiles = Math.floor(completedRuns / runsPerFile);
+    const scheduledFiles = this.benchmarks.length;
+    const completedRunsForFile = finished ? runsPerFile :
+                                 completedRuns % runsPerFile;
+    const completedConfig = this.completedConfig;
+    const scheduledConfig = this.scheduledConfig;
+
+    // Calculate the percentage.
+    let runRate = 0;  // Rate of current incomplete run.
+    if (completedConfig !== scheduledConfig) {
+      runRate = completedConfig / scheduledConfig;
+    }
+    const completedRate = ((completedRuns + runRate) / scheduledRuns);
+    const percent = pad(Math.floor(completedRate * 100), 3, ' ');
+
+    const caption = finished ? 'Done\n' : this.currentFile;
+    return `[${getTime(diff)}|% ${percent}` +
+          `| ${fraction(completedFiles, scheduledFiles)} files ` +
+          `| ${fraction(completedRunsForFile, runsPerFile)} runs ` +
+          `| ${fraction(completedConfig, scheduledConfig)} configs]` +
+          `: ${caption}`;
+  }
+
+  updateProgress(finished) {
+    if (!process.stderr.isTTY || process.stdout.isTTY) {
+      return;
+    }
+    readline.clearLine(process.stderr);
+    readline.cursorTo(process.stderr, 0);
+    process.stderr.write(this.getProgress());
+  }
+}
+
+module.exports = BenchmarkProgress;

benchmark/_cli.js

@@ -45,13 +45,13 @@ function CLI(usage, settings) {
         currentOptional = arg.slice(1);
       }
 
-      // Default the value to true
-      if (!settings.arrayArgs.includes(currentOptional)) {
+      if (settings.boolArgs && settings.boolArgs.includes(currentOptional)) {
         this.optional[currentOptional] = true;
+        mode = 'both';
+      } else {
+        // expect the next value to be option related (either -- or the value)
+        mode = 'option';
       }
-
-      // expect the next value to be option related (either -- or the value)
-      mode = 'option';
     } else if (mode === 'option') {
       // Optional arguments value
 

benchmark/common.js

@@ -128,6 +128,14 @@ Benchmark.prototype.http = function(options, cb) {
 
 Benchmark.prototype._run = function() {
   const self = this;
+  // If forked, report to the parent.
+  if (process.send) {
+    process.send({
+      type: 'config',
+      name: this.name,
+      queueLength: this.queue.length
+    });
+  }
 
   (function recursive(queueIndex) {
     const config = self.queue[queueIndex];
@@ -217,7 +225,8 @@ Benchmark.prototype.report = function(rate, elapsed) {
     name: this.name,
     conf: this.config,
     rate: rate,
-    time: elapsed[0] + elapsed[1] / 1e9
+    time: elapsed[0] + elapsed[1] / 1e9,
+    type: 'report'
   });
 };
 

benchmark/compare.js

@@ -3,6 +3,7 @@
 const fork = require('child_process').fork;
 const path = require('path');
 const CLI = require('./_cli.js');
+const BenchmarkProgress = require('./_benchmark_progress.js');
 
 //
 // Parse arguments
@@ -13,13 +14,15 @@ const cli = CLI(`usage: ./node compare.js [options] [--] <category> ...
   The output is formatted as csv, which can be processed using for
   example 'compare.R'.
 
-  --new    ./new-node-binary  new node binary (required)
-  --old    ./old-node-binary  old node binary (required)
-  --runs   30                 number of samples
-  --filter pattern            string to filter benchmark scripts
-  --set    variable=value     set benchmark variable (can be repeated)
+  --new      ./new-node-binary  new node binary (required)
+  --old      ./old-node-binary  old node binary (required)
+  --runs     30                 number of samples
+  --filter   pattern            string to filter benchmark scripts
+  --set      variable=value     set benchmark variable (can be repeated)
+  --no-progress                 don't show benchmark progress indicator
 `, {
-  arrayArgs: ['set']
+  arrayArgs: ['set'],
+  boolArgs: ['no-progress']
 });
 
 if (!cli.optional.new || !cli.optional.old) {
@@ -39,6 +42,9 @@ if (benchmarks.length === 0) {
 
 // Create queue from the benchmarks list such both node versions are tested
 // `runs` amount of times each.
+// Note: BenchmarkProgress relies on this order to estimate
+// how much runs remaining for a file. All benchmarks generated from
+// the same file must be run consecutively.
 const queue = [];
 for (const filename of benchmarks) {
   for (let iter = 0; iter < runs; iter++) {
@@ -47,10 +53,20 @@ for (const filename of benchmarks) {
     }
   }
 }
+// queue.length = binary.length * runs * benchmarks.length
 
 // Print csv header
 console.log('"binary", "filename", "configuration", "rate", "time"');
 
+const kStartOfQueue = 0;
+
+const showProgress = !cli.optional['no-progress'];
+let progress;
+if (showProgress) {
+  progress = new BenchmarkProgress(queue, benchmarks);
+  progress.startQueue(kStartOfQueue);
+}
+
 (function recursive(i) {
   const job = queue[i];
 
@@ -59,29 +75,40 @@ console.log('"binary", "filename", "configuration", "rate", "time"');
   });
 
   child.on('message', function(data) {
-    // Construct configuration string, " A=a, B=b, ..."
-    let conf = '';
-    for (const key of Object.keys(data.conf)) {
-      conf += ' ' + key + '=' + JSON.stringify(data.conf[key]);
-    }
-    conf = conf.slice(1);
+    if (data.type === 'report') {
+      // Construct configuration string, " A=a, B=b, ..."
+      let conf = '';
+      for (const key of Object.keys(data.conf)) {
+        conf += ' ' + key + '=' + JSON.stringify(data.conf[key]);
+      }
+      conf = conf.slice(1);
+      // Escape quotes (") for correct csv formatting
+      conf = conf.replace(/"/g, '""');
 
-    // Escape quotes (") for correct csv formatting
-    conf = conf.replace(/"/g, '""');
-
-    console.log(`"${job.binary}", "${job.filename}", "${conf}", ` +
-                `${data.rate}, ${data.time}`);
+      console.log(`"${job.binary}", "${job.filename}", "${conf}", ` +
+                  `${data.rate}, ${data.time}`);
+      if (showProgress) {
+        // One item in the subqueue has been completed.
+        progress.completeConfig(data);
+      }
+    } else if (showProgress && data.type === 'config') {
+      // The child has computed the configurations, ready to run subqueue.
+      progress.startSubqueue(data, i);
+    }
   });
 
   child.once('close', function(code) {
     if (code) {
       process.exit(code);
       return;
     }
+    if (showProgress) {
+      progress.completeRun(job);
+    }
 
     // If there are more benchmarks execute the next
     if (i + 1 < queue.length) {
       recursive(i + 1);
     }
   });
-})(0);
+})(kStartOfQueue);

benchmark/run.js

@@ -44,6 +44,9 @@ if (format === 'csv') {
   }
 
   child.on('message', function(data) {
+    if (data.type !== 'report') {
+      return;
+    }
     // Construct configuration string, " A=a, B=b, ..."
     let conf = '';
     for (const key of Object.keys(data.conf)) {

benchmark/scatter.js

@@ -42,6 +42,10 @@ function csvEncodeValue(value) {
   const child = fork(path.resolve(__dirname, filepath), cli.optional.set);
 
   child.on('message', function(data) {
+    if (data.type !== 'report') {
+      return;
+    }
+
     // print csv header
     if (printHeader) {
       const confHeader = Object.keys(data.conf)