feat(@angular-devkit/benchmark): add package

Author: filipesilva

.monorepo.json

@@ -87,6 +87,11 @@
       "hash": "9d2161b7ca9044c2286c7d66e10ead12",
       "snapshotRepo": "angular/angular-devkit-architect-cli-builds"
     },
+    "@angular-devkit/benchmark": {
+      "name": "Benchmark",
+      "section": "Tooling",
+      "version": "0.0.0"
+    },
     "@angular-devkit/build-optimizer": {
       "name": "Build Optimizer",
       "links": [

README.md

@@ -51,6 +51,7 @@ will also be available to third party tools and IDEs.
 | Project | Package | Version | Links |
 |---|---|---|---|
 **Angular CLI** | [`@angular/cli`](https://npmjs.com/package/@angular/cli) | [![latest](https://img.shields.io/npm/v/%40angular%2Fcli/latest.svg)](https://npmjs.com/package/@angular/cli) | [![README](https://img.shields.io/badge/README--green.svg)](https://github.com/angular/angular-cli/blob/master/packages/angular/cli/README.md) [![snapshot](https://img.shields.io/badge/snapshot--blue.svg)](https://github.com/angular/cli-builds)
+**Benchmark** | [`@angular-devkit/benchmark`](https://npmjs.com/package/@angular-devkit/benchmark) | [![latest](https://img.shields.io/npm/v/%40angular-devkit%2Fbenchmark/latest.svg)](https://npmjs.com/package/@angular-devkit/benchmark) | 
 **Schematics CLI** | [`@angular-devkit/schematics-cli`](https://npmjs.com/package/@angular-devkit/schematics-cli) | [![latest](https://img.shields.io/npm/v/%40angular-devkit%2Fschematics-cli/latest.svg)](https://npmjs.com/package/@angular-devkit/schematics-cli) |  [![snapshot](https://img.shields.io/badge/snapshot--blue.svg)](https://github.com/angular/angular-devkit-schematics-cli-builds)
 
 

bin/benchmark

@@ -0,0 +1,14 @@
+#!/usr/bin/env node
+/**
+ * @license
+ * Copyright Google Inc. All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+'use strict';
+
+
+require('../lib/bootstrap-local');
+const packages = require('../lib/packages').packages;
+require(packages['@angular-devkit/benchmark'].bin['benchmark']);

package.json

@@ -5,6 +5,7 @@
   "description": "Software Development Kit for Angular",
   "bin": {
     "architect": "./bin/architect",
+    "benchmark": "./bin/benchmark",
     "build-optimizer": "./bin/build-optimizer",
     "devkit-admin": "./bin/devkit-admin",
     "ng": "./bin/ng",

packages/angular_devkit/benchmark/BUILD

@@ -0,0 +1,57 @@
+# Copyright Google Inc. All Rights Reserved.
+#
+# Use of this source code is governed by an MIT-style license that can be
+# found in the LICENSE file at https://angular.io/license
+
+licenses(["notice"])  # MIT
+
+load("@build_bazel_rules_typescript//:defs.bzl", "ts_library")
+load("@build_bazel_rules_nodejs//:defs.bzl", "jasmine_node_test")
+
+package(default_visibility = ["//visibility:public"])
+
+ts_library(
+    name = "benchmark",
+    srcs = glob(
+        include = ["src/**/*.ts"],
+        exclude = [
+            "src/**/*_spec.ts",
+            "src/**/*_spec_large.ts",
+            "src/**/*_benchmark.ts",
+        ],
+    ),
+    module_name = "@angular-devkit/benchmark",
+    module_root = "src/index.d.ts",
+    deps = [
+        "//packages/angular_devkit/core",
+        "@rxjs",
+        "@rxjs//operators",
+        # @typings: node
+    ],
+)
+
+ts_library(
+    name = "benchmark_test_lib",
+    srcs = glob(
+        include = [
+            "src/**/*_spec.ts",
+            "src/**/*_spec_large.ts",
+        ],
+    ),
+    data = [
+        "src/test/test-script.js"
+    ],
+    deps = [
+        ":benchmark",
+        "//packages/angular_devkit/core",
+        "@rxjs",
+        "@rxjs//operators",
+        # @typings: jasmine
+        # @typings: node
+    ],
+)
+
+jasmine_node_test(
+    name = "benchmark_test",
+    srcs = [":benchmark_test_lib"],
+)

packages/angular_devkit/benchmark/README.md

@@ -0,0 +1,113 @@
+# Angular Devkit Benchmark
+
+This tools provides benchmark information for processes.
+The tool will gathering metrics from a given command, then average them out over the total runs.
+
+It currently shows only time, process, cpu, and memory used but might be extended in the future.
+
+This tool was created to provide an objective and reproducible way of benchmarking process
+performance.
+
+Given a process (or its source code), process inputs and environment, keeping two of these elements
+constant while varying the third should allow for meaningful benchmarks over time.
+
+In the context of the DevKit, we publish many CLI tools and have access to their source code.
+By tracking tool resource usage we can catch performance regressions or improvements on our CI.
+
+
+## Installation
+
+You can install the benchmark tool via `npm install -g benchmark` for a global install, or without
+`-g` for a local install.
+Installing globally gives you access to the `benchmark` binary in your `PATH`.
+
+
+## CLI Usage
+
+Call the `benchmark` binary, followed by options, then double dash, then the command to benchmark:
+
+```
+$ benchmark -- node fibonacci.js 40
+[benchmark] Benchmarking process over 5 iterations, with up to 5 retries.
+[benchmark]   node fibonacci.js 40 (at D:\sandbox\latest-project)
+[benchmark] Process Stats
+[benchmark]   Elapsed Time: 2365.40 ms (2449.00, 2444.00, 2357.00, 2312.00, 2265.00)
+[benchmark]   Average Process usage: 1.00 process(es) (1.00, 1.00, 1.00, 1.00, 1.00)
+[benchmark]   Peak Process usage: 1.00 process(es) (1.00, 1.00, 1.00, 1.00, 1.00)
+[benchmark]   Average CPU usage: 4.72 % (5.03, 4.86, 4.50, 4.50, 4.69)
+[benchmark]   Peak CPU usage: 23.40 % (25.00, 23.40, 21.80, 21.80, 25.00)
+[benchmark]   Average Memory usage: 22.34 MB (22.32, 22.34, 22.34, 22.35, 22.35)
+[benchmark]   Peak Memory usage: 22.34 MB (22.32, 22.34, 22.34, 22.35, 22.35)
+```
+
+For more information on the available options, run `benchmark --help`:
+```
+$ benchmark --help
+[benchmark] benchmark [options] -- [command to benchmark]
+
+Collects process stats from running the command.
+
+Options:
+    --help                    Show this message.
+    --verbose                 Show more information while running.
+    --exit-code               Expected exit code for the command. Default is 0.
+    --iterations              Number of iterations to run the benchmark over. Default is 5.
+    --retries                 Number of times to retry when process fails. Default is 5.
+    --cwd                     Current working directory to run the process in.
+    --output-file             File to output benchmark log to.
+    --overwrite-output-file   If the output file should be overwritten rather than appended to.
+    --prefix                  Logging prefix.
+
+Example:
+    benchmark --iterations=3 -- node my-script.js
+```
+
+
+NB: the `fibonacci.js` in the example is a naive implementation of a fibonacci number calculator:
+```
+// fibonacci.js
+const fib = (n) => n > 1 ? fib(n - 1) + fib(n - 2) : n;
+console.log(fib(parseInt(process.argv[2])));
+```
+
+## API Usage
+
+You can also use the benchmarking API directly:
+
+```
+import { Command, defaultStatsCapture, runBenchmark } from '@angular-devkit/benchmark';
+
+const command = new Command('node', ['fibonacci.js', '40']);
+const captures = [defaultStatsCapture];
+
+runBenchmark({ command, command }).subscribe(results => {
+  // results is:[{
+  //   "name": "Process Stats",
+  //   "metrics": [{
+  //     "name": "Elapsed Time", "unit": "ms", "value": 1883.6,
+  //     "componentValues": [1733, 1957, 1580, 1763, 2385]
+  //   }, {
+  //     "name": "Average Process usage", "unit": "process(es)", "value": 1,
+  //     "componentValues": [1, 1, 1, 1, 1]
+  //   }, {
+  //     "name": "Peak Process usage", "unit": "process(es)", "value": 1,
+  //     "componentValues": [1, 1, 1, 1, 1]
+  //   }, {
+  //     "name": "Average CPU usage", "unit": "%", "value": 3.0855555555555556,
+  //     "componentValues": [1.9625, 1.9500000000000002, 1.9500000000000002, 4.887499999999999, 4.677777777777778]
+  //   }, {
+  //     "name": "Peak CPU usage", "unit": "%", "value": 19.380000000000003,
+  //     "componentValues": [15.7, 15.6, 15.6, 25, 25]
+  //   }, {
+  //     "name": "Average Memory usage", "unit": "MB", "value": 22.364057600000002,
+  //     "componentValues": [22.383104, 22.332416, 22.401024, 22.355968, 22.347776]
+  //   }, {
+  //     "name": "Peak Memory usage", "unit": "MB", "value": 22.3649792,
+  //     "componentValues": [22.384639999999997, 22.335487999999998, 22.401024, 22.355968, 22.347776]
+  //   }]
+  // }]
+});
+```
+
+A good example of API usage is the `benchmark` binary itself, found in `./bin/benchmark.ts`.
+We recommend using TypeScript to get full access to the interfaces included.

packages/angular_devkit/benchmark/bin/benchmark.ts

@@ -0,0 +1,140 @@
+#!/usr/bin/env node
+/**
+ * @license
+ * Copyright Google Inc. All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+
+import { logging, tags, terminal } from '@angular-devkit/core';
+import { appendFileSync, writeFileSync } from 'fs';
+import * as minimist from 'minimist';
+import { filter, map } from 'rxjs/operators';
+import { Command } from '../src/command';
+import { defaultReporter } from '../src/default-reporter';
+import { defaultStatsCapture } from '../src/default-stats-capture';
+import { runBenchmark } from '../src/run-benchmark';
+
+// Show usage of the CLI tool, and exit the process.
+function usage(logger: logging.Logger, exitCode = 0): never {
+  logger.info(tags.stripIndent`
+    benchmark [options] -- [command to benchmark]
+
+    Collects process stats from running the command.
+
+    Options:
+        --help                    Show this message.
+        --verbose                 Show more information while running.
+        --exit-code               Expected exit code for the command. Default is 0.
+        --iterations              Number of iterations to run the benchmark over. Default is 5.
+        --retries                 Number of times to retry when process fails. Default is 5.
+        --cwd                     Current working directory to run the process in.
+        --output-file             File to output benchmark log to.
+        --overwrite-output-file   If the output file should be overwritten rather than appended to.
+        --prefix                  Logging prefix.
+
+    Example:
+        benchmark --iterations=3 -- node my-script.js
+  `);
+
+  process.exit(exitCode);
+  // The node typing sometimes don't have a never type for process.exit().
+  throw 0;
+}
+
+interface BenchmarkCliArgv {
+  help: boolean;
+  verbose: boolean;
+  'overwrite-output-file': boolean;
+  'exit-code': number;
+  iterations: number;
+  retries: number;
+  'output-file': string | null;
+  cwd: string;
+  prefix: string;
+  '--': string[] | null;
+}
+
+// Parse the command line.
+const argv = minimist(process.argv.slice(2), {
+  boolean: ['help', 'verbose', 'overwrite-output-file'],
+  default: {
+    'exit-code': 0,
+    'iterations': 5,
+    'retries': 5,
+    'output-file': null,
+    'cwd': process.cwd(),
+    'prefix': '[benchmark] ',
+  },
+  '--': true,
+}) as {} as BenchmarkCliArgv;
+
+// Create the DevKit Logger used through the CLI.
+const logger = new logging.TransformLogger(
+  'benchmark-prefix-logger',
+  stream => stream.pipe(map(entry => (entry.message = argv['prefix'] + entry.message, entry))),
+);
+
+// Log to console.
+logger
+  .pipe(filter(entry => (entry.level != 'debug' || argv['verbose'])))
+  .subscribe(entry => {
+    let color: (s: string) => string = x => terminal.dim(terminal.white(x));
+    let output = process.stdout;
+    switch (entry.level) {
+      case 'info':
+        color = terminal.white;
+        break;
+      case 'warn':
+        color = terminal.yellow;
+        break;
+      case 'error':
+        color = terminal.red;
+        output = process.stderr;
+        break;
+      case 'fatal':
+        color = (x: string) => terminal.bold(terminal.red(x));
+        output = process.stderr;
+        break;
+    }
+
+    output.write(color(entry.message) + '\n');
+  });
+
+
+// Print help.
+if (argv['help']) {
+  usage(logger);
+}
+
+// Exit early if we can't find the command to benchmark.
+if (!Array.isArray(argv['--']) || (argv['--'] as Array<string>).length < 1) {
+  logger.fatal(`Missing command, see benchmark --help for help.`);
+  process.exit(1);
+  throw 0;
+}
+
+// Setup file logging.
+if (argv['output-file'] !== null) {
+  if (argv['overwrite-output-file']) {
+    writeFileSync(argv['output-file'] as string, '');
+  }
+  logger.pipe(filter(entry => (entry.level != 'debug' || argv['verbose'])))
+    .subscribe(entry => appendFileSync(argv['output-file'] as string, `${entry.message}\n`));
+}
+
+// Run benchmark on given command, capturing stats and reporting them.
+const exitCode = argv['exit-code'];
+const cmd = (argv['--'] as Array<string>)[0];
+const args = (argv['--'] as Array<string>).slice(1);
+const command = new Command(cmd, args, argv['cwd'], exitCode);
+const captures = [defaultStatsCapture];
+const reporters = [defaultReporter(logger)];
+const iterations = argv['iterations'];
+const retries = argv['retries'];
+
+logger.info(`Benchmarking process over ${iterations} iterations, with up to ${retries} retries.`);
+logger.info(`  ${command.toString()}`);
+
+runBenchmark({ command, captures, reporters, iterations, retries, logger }).subscribe();

packages/angular_devkit/benchmark/package.json

@@ -0,0 +1,23 @@
+{
+  "name": "@angular-devkit/benchmark",
+  "version": "0.0.0",
+  "description": "Angular Benchmark",
+  "bin": {
+    "benchmark": "./bin/benchmark.js"
+  },
+  "keywords": [
+    "benchmark"
+  ],
+  "engines": {
+    "node": ">= 8.9.0",
+    "npm": ">= 5.5.1"
+  },
+  "dependencies": {
+    "@angular-devkit/core": "0.0.0",
+    "minimist": "^1.2.0",
+    "pidusage": "^2.0.16",
+    "pidtree": "^0.3.0",
+    "rxjs": "~6.2.0",
+    "tree-kill": "^1.2.0"
+  }
+}