Add benchmark package and AIO benchmark

.monorepo.json

@@ -87,6 +87,11 @@
       "hash": "51b6cc55b25fce87f6a92f0d0942f79c",
       "snapshotRepo": "angular/angular-devkit-architect-cli-builds"
     },
+    "@angular-devkit/benchmark": {
+      "name": "Benchmark",
+      "section": "Tooling",
+      "version": "0.0.0"
+    },
     "@angular-devkit/build-optimizer": {
       "name": "Build Optimizer",
       "links": [

benchmark/aio/.gitignore

@@ -0,0 +1 @@
+angular/

benchmark/aio/package.json

@@ -0,0 +1,18 @@
+{
+  "name": "aio-benchmark",
+  "version": "1.0.0",
+  "description": "",
+  "main": "index.js",
+  "scripts": {
+    "initialize": "yarn clone && yarn setup && yarn update", 
+    "clone": "(git clone https://github.com/angular/angular --depth 1 || true) && cd angular && git fetch origin dd2a650c3455f3bc0a88f8181758a84aacb25fea && git checkout -f FETCH_HEAD",
+    "setup": "cd angular && yarn && cd aio && yarn && yarn setup",
+    "update": "cd angular/aio && yarn add ../../../../dist/_angular-devkit_build-angular.tgz --dev",
+    "//": "Shouldn't need to install the package twice, but the first install seems to leave two @ngtools/webpack installs around.",
+    "postupdate": "cd angular/aio && yarn add ../../../../dist/_angular-devkit_build-angular.tgz --dev",
+    "benchmark": "cd angular/aio && benchmark --verbose -- yarn ~~build --configuration=stable"
+  },
+  "keywords": [],
+  "author": "",
+  "license": "ISC"
+}

bin/benchmark

@@ -0,0 +1,19 @@
+#!/usr/bin/env node
+/**
+ * @license
+ * Copyright Google Inc. All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+'use strict';
+
+
+require('../lib/bootstrap-local');
+const packages = require('../lib/packages').packages;
+const main = require(packages['@angular-devkit/benchmark'].bin['benchmark']).main;
+
+const args = process.argv.slice(2);
+main({ args })
+  .then(exitCode => process.exitCode = exitCode)
+  .catch(e => { throw (e); });

lib/packages.ts

@@ -212,7 +212,9 @@ export const packages: PackageMap =
         relative: path.relative(path.dirname(__dirname), pkgRoot),
         main: path.resolve(pkgRoot, 'src/index.ts'),
         private: packageJson.private,
-        tar: path.join(distRoot, name.replace('/', '_') + '.tgz'),
+        // yarn doesn't take kindly to @ in tgz filenames
+        // https://github.com/yarnpkg/yarn/issues/6339
+        tar: path.join(distRoot, name.replace(/\/|@/g, '_') + '.tgz'),
         bin,
         name,
         packageJson,

package.json

@@ -5,6 +5,7 @@
   "description": "Software Development Kit for Angular",
   "bin": {
     "architect": "./bin/architect",
+    "benchmark": "./bin/benchmark",
     "build-optimizer": "./bin/build-optimizer",
     "devkit-admin": "./bin/devkit-admin",
     "ng": "./bin/ng",

packages/angular_devkit/benchmark/BUILD

@@ -0,0 +1,59 @@
+# Copyright Google Inc. All Rights Reserved.
+#
+# Use of this source code is governed by an MIT-style license that can be
+# found in the LICENSE file at https://angular.io/license
+
+licenses(["notice"])  # MIT
+
+load("@build_bazel_rules_typescript//:defs.bzl", "ts_library")
+load("@build_bazel_rules_nodejs//:defs.bzl", "jasmine_node_test")
+
+package(default_visibility = ["//visibility:public"])
+
+ts_library(
+    name = "benchmark",
+    srcs = glob(
+        include = ["src/**/*.ts"],
+        exclude = [
+            "src/**/*_spec.ts",
+            "src/**/*_spec_large.ts",
+            "src/**/*_benchmark.ts",
+        ],
+    ),
+    module_name = "@angular-devkit/benchmark",
+    module_root = "src/index.d.ts",
+    deps = [
+        "//packages/angular_devkit/core",
+        "@rxjs",
+        "@rxjs//operators",
+        # @typings: node
+    ],
+)
+
+ts_library(
+    name = "benchmark_test_lib",
+    srcs = glob(
+        include = [
+            "src/**/*_spec.ts",
+            "src/**/*_spec_large.ts",
+        ],
+    ),
+    data = [
+        "src/test/exit-code-one.js",
+        "src/test/fibonacci.js",
+        "src/test/test-script.js",
+    ],
+    deps = [
+        ":benchmark",
+        "//packages/angular_devkit/core",
+        "@rxjs",
+        "@rxjs//operators",
+        # @typings: jasmine
+        # @typings: node
+    ],
+)
+
+jasmine_node_test(
+    name = "benchmark_test",
+    srcs = [":benchmark_test_lib"],
+)

packages/angular_devkit/benchmark/README.md

@@ -0,0 +1,116 @@
+# Angular Devkit Benchmark
+
+This tool provides benchmark information for processes.
+The tool will gathering metrics from a given command, then average them out over the total runs.
+
+It currently shows only time, process, cpu, and memory used but might be extended in the future.
+
+This tool was created to provide an objective and reproducible way of benchmarking process
+performance.
+
+Given a process (or its source code), process inputs and environment, keeping two of these elements
+constant while varying the third should allow for meaningful benchmarks over time.
+
+In the context of the DevKit, we publish many CLI tools and have access to their source code.
+By tracking tool resource usage we can catch performance regressions or improvements on our CI.
+
+
+## STABILITY AND SUPPORT DISCLAIMER
+
+This package is not currently stable. Usage, output and API may change at any time.
+Support is not ensured.
+
+## Installation
+
+You can install the benchmark tool via `npm install -g benchmark` for a global install, or without
+`-g` for a local install.
+Installing globally gives you access to the `benchmark` binary in your `PATH`.
+
+
+## CLI Usage
+
+Call the `benchmark` binary, followed by options, then double dash, then the command to benchmark.
+
+For more information on the available options, run `benchmark --help`:
+```
+$ benchmark --help
+[benchmark] benchmark [options] -- [command to benchmark]
+
+Collects process stats from running the command.
+
+Options:
+    --help                    Show this message.
+    (... other available options)
+
+Example:
+    benchmark --iterations=3 -- node my-script.js
+```
+
+
+## Example
+
+Given the naive implementation of a fibonacci number calculator below:
+```
+// fibonacci.js
+const fib = (n) => n > 1 ? fib(n - 1) + fib(n - 2) : n;
+console.log(fib(parseInt(process.argv[2])));
+```
+
+Run `benchmark -- node fibonacci.js 40` to benchmark calculating the 40th fibonacci number:
+
+```
+$ benchmark -- node fibonacci.js 40
+[benchmark] Benchmarking process over 5 iterations, with up to 5 retries.
+[benchmark]   node fibonacci.js 40 (at D:\sandbox\latest-project)
+[benchmark] Process Stats
+[benchmark]   Elapsed Time: 2365.40 ms (2449.00, 2444.00, 2357.00, 2312.00, 2265.00)
+[benchmark]   Average Process usage: 1.00 process(es) (1.00, 1.00, 1.00, 1.00, 1.00)
+[benchmark]   Peak Process usage: 1.00 process(es) (1.00, 1.00, 1.00, 1.00, 1.00)
+[benchmark]   Average CPU usage: 4.72 % (5.03, 4.86, 4.50, 4.50, 4.69)
+[benchmark]   Peak CPU usage: 23.40 % (25.00, 23.40, 21.80, 21.80, 25.00)
+[benchmark]   Average Memory usage: 22.34 MB (22.32, 22.34, 22.34, 22.35, 22.35)
+[benchmark]   Peak Memory usage: 22.34 MB (22.32, 22.34, 22.34, 22.35, 22.35)
+```
+
+
+## API Usage
+
+You can also use the benchmarking API directly:
+
+```
+import { Command, defaultStatsCapture, runBenchmark } from '@angular-devkit/benchmark';
+
+const command = new Command('node', ['fibonacci.js', '40']);
+const captures = [defaultStatsCapture];
+
+runBenchmark({ command, command }).subscribe(results => {
+  // results is:[{
+  //   "name": "Process Stats",
+  //   "metrics": [{
+  //     "name": "Elapsed Time", "unit": "ms", "value": 1883.6,
+  //     "componentValues": [1733, 1957, 1580, 1763, 2385]
+  //   }, {
+  //     "name": "Average Process usage", "unit": "process(es)", "value": 1,
+  //     "componentValues": [1, 1, 1, 1, 1]
+  //   }, {
+  //     "name": "Peak Process usage", "unit": "process(es)", "value": 1,
+  //     "componentValues": [1, 1, 1, 1, 1]
+  //   }, {
+  //     "name": "Average CPU usage", "unit": "%", "value": 3.0855555555555556,
+  //     "componentValues": [1.9625, 1.9500000000000002, 1.9500000000000002, 4.887499999999999, 4.677777777777778]
+  //   }, {
+  //     "name": "Peak CPU usage", "unit": "%", "value": 19.380000000000003,
+  //     "componentValues": [15.7, 15.6, 15.6, 25, 25]
+  //   }, {
+  //     "name": "Average Memory usage", "unit": "MB", "value": 22.364057600000002,
+  //     "componentValues": [22.383104, 22.332416, 22.401024, 22.355968, 22.347776]
+  //   }, {
+  //     "name": "Peak Memory usage", "unit": "MB", "value": 22.3649792,
+  //     "componentValues": [22.384639999999997, 22.335487999999998, 22.401024, 22.355968, 22.347776]
+  //   }]
+  // }]
+});
+```
+
+A good example of API usage is the `main` binary itself, found in `./src/main.ts`.
+We recommend using TypeScript to get full access to the interfaces included.

packages/angular_devkit/benchmark/package.json

@@ -0,0 +1,24 @@
+{
+  "name": "@angular-devkit/benchmark",
+  "version": "0.0.0",
+  "private": true,
+  "description": "Angular Benchmark",
+  "bin": {
+    "benchmark": "./src/main.js"
+  },
+  "keywords": [
+    "benchmark"
+  ],
+  "engines": {
+    "node": ">= 8.9.0",
+    "npm": ">= 5.5.1"
+  },
+  "dependencies": {
+    "@angular-devkit/core": "0.0.0",
+    "minimist": "^1.2.0",
+    "pidusage": "^2.0.16",
+    "pidtree": "^0.3.0",
+    "rxjs": "~6.2.0",
+    "tree-kill": "^1.2.0"
+  }
+}

packages/angular_devkit/benchmark/src/command.ts

@@ -0,0 +1,23 @@
+/**
+ * @license
+ * Copyright Google Inc. All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+
+export class Command {
+  constructor(
+    public cmd: string,
+    public args: string[] = [],
+    public cwd: string = process.cwd(),
+    public expectedExitCode = 0,
+  ) { }
+
+  toString() {
+    const { cmd, args, cwd } = this;
+    const argsStr = args.length > 0 ? ' ' + args.join(' ') : '';
+
+    return `${cmd}${argsStr} (at ${cwd})`;
+  }
+}

packages/angular_devkit/benchmark/src/default-reporter.ts

@@ -0,0 +1,24 @@
+/**
+ * @license
+ * Copyright Google Inc. All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+import { logging, tags } from '@angular-devkit/core';
+import { AggregatedMetric, BenchmarkReporter, Metric } from './interfaces';
+
+export const defaultReporter = (logger: logging.Logger): BenchmarkReporter => (process, groups) => {
+  const toplevelLogger = logger;
+  const indentLogger = new logging.IndentLogger('benchmark-indent-logger', toplevelLogger);
+
+  const formatMetric = (metric: Metric | AggregatedMetric) => tags.oneLine`
+    ${metric.name}: ${metric.value.toFixed(2)} ${metric.unit}
+    ${metric.componentValues ? `(${metric.componentValues.map(v => v.toFixed(2)).join(', ')})` : ''}
+  `;
+
+  groups.forEach(group => {
+    toplevelLogger.info(`${group.name}`);
+    group.metrics.forEach(metric => indentLogger.info(formatMetric(metric)));
+  });
+};

packages/angular_devkit/benchmark/src/default-stats-capture.ts

@@ -0,0 +1,60 @@
+/**
+ * @license
+ * Copyright Google Inc. All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+
+import { Observable } from 'rxjs';
+import { map, reduce } from 'rxjs/operators';
+import { AggregatedProcessStats, Capture, MetricGroup, MonitoredProcess } from './interfaces';
+import { cumulativeMovingAverage, max } from './utils';
+
+
+export const defaultStatsCapture: Capture = (
+  process: MonitoredProcess,
+): Observable<MetricGroup> => {
+  type Accumulator = {
+    elapsed: number,
+    avgProcesses: number,
+    peakProcesses: number,
+    avgCpu: number,
+    peakCpu: number,
+    avgMemory: number,
+    peakMemory: number,
+  };
+  const seed: Accumulator = {
+    elapsed: 0,
+    avgProcesses: 0,
+    peakProcesses: 0,
+    avgCpu: 0,
+    peakCpu: 0,
+    avgMemory: 0,
+    peakMemory: 0,
+  };
+
+  return process.stats$.pipe(
+    reduce<AggregatedProcessStats, Accumulator>((acc, val, idx) => ({
+      elapsed: val.elapsed,
+      avgProcesses: cumulativeMovingAverage(acc.avgProcesses, val.processes, idx),
+      peakProcesses: max(acc.peakProcesses, val.processes),
+      avgCpu: cumulativeMovingAverage(acc.avgCpu, val.cpu, idx),
+      peakCpu: max(acc.peakCpu, val.cpu),
+      avgMemory: cumulativeMovingAverage(acc.avgMemory, val.memory, idx),
+      peakMemory: max(acc.peakMemory, val.memory),
+    }), seed),
+    map(metrics => ({
+      name: 'Process Stats',
+      metrics: [
+        { name: 'Elapsed Time', unit: 'ms', value: metrics.elapsed },
+        { name: 'Average Process usage', unit: 'process(es)', value: metrics.avgProcesses },
+        { name: 'Peak Process usage', unit: 'process(es)', value: metrics.peakProcesses },
+        { name: 'Average CPU usage', unit: '%', value: metrics.avgCpu },
+        { name: 'Peak CPU usage', unit: '%', value: metrics.peakCpu },
+        { name: 'Average Memory usage', unit: 'MB', value: metrics.avgMemory * 1e-6 },
+        { name: 'Peak Memory usage', unit: 'MB', value: metrics.peakMemory * 1e-6 },
+      ],
+    })),
+  );
+};