feat(@angular-devkit/build-angular): Switch to karma-coverage

This commit switches coverage tooling from karma-coverage-istanbul-reporter
to karma-coverage since it's better supported.

Closes #17757

Author: kyliau

integration/angular_cli/karma.conf.js

@@ -12,16 +12,19 @@ module.exports = function (config) {
       require('karma-jasmine'),
       require('karma-chrome-launcher'),
       require('karma-jasmine-html-reporter'),
-      require('karma-coverage-istanbul-reporter'),
+      require('karma-coverage'),
       require('@angular-devkit/build-angular/plugins/karma'),
     ],
     client: {
       clearContext: false, // leave Jasmine Spec Runner output visible in browser
     },
-    coverageIstanbulReporter: {
+    coverageReporter: {
       dir: path.join(__dirname, 'coverage'),
-      reports: ['html', 'lcovonly'],
-      fixWebpackSourcePaths: true,
+      subdir: '.',
+      reporters: [
+        {type: 'html'},
+        {type: 'lcov'},
+      ],
     },
     reporters: ['progress', 'kjhtml'],
     port: 9876,

integration/angular_cli/package.json

@@ -36,7 +36,7 @@
     "jasmine-spec-reporter": "~5.0.0",
     "karma": "~4.4.1",
     "karma-chrome-launcher": "~3.1.0",
-    "karma-coverage-istanbul-reporter": "~2.1.0",
+    "karma-coverage": "~2.0.3",
     "karma-jasmine": "~3.3.0",
     "karma-jasmine-html-reporter": "^1.5.0",
     "protractor": "~5.4.3",

package.json

@@ -161,7 +161,7 @@
     "jsonc-parser": "2.3.0",
     "karma": "~5.2.0",
     "karma-chrome-launcher": "~3.1.0",
-    "karma-coverage-istanbul-reporter": "~3.0.0",
+    "karma-coverage": "~2.0.3",
     "karma-jasmine": "~4.0.0",
     "karma-jasmine-html-reporter": "^1.5.0",
     "karma-source-map-support": "1.4.0",

packages/angular_devkit/build_angular/BUILD.bazel

@@ -282,7 +282,7 @@ LARGE_SPECS = {
         "extra_deps": [
             "@npm//karma",
             "@npm//karma-chrome-launcher",
-            "@npm//karma-coverage-istanbul-reporter",
+            "@npm//karma-coverage",
             "@npm//karma-jasmine",
             "@npm//karma-jasmine-html-reporter",
             "@npm//puppeteer",

packages/angular_devkit/build_angular/src/angular-cli-files/plugins/karma.ts

@@ -88,12 +88,32 @@ const init: any = (config: any, emitter: any, customFileHandlers: any) => {
 
   config.reporters.unshift('@angular-devkit/build-angular--event-reporter');
 
-  // When using code-coverage, auto-add coverage-istanbul.
-  config.reporters = config.reporters || [];
-  if (options.codeCoverage && config.reporters.indexOf('coverage-istanbul') === -1) {
-    config.reporters.unshift('coverage-istanbul');
+  // When using code-coverage, auto-add karma-coverage.
+  if (options.codeCoverage) {
+    config.plugins = config.plugins || [];
+    config.reporters = config.reporters || [];
+    const {plugins, reporters} = config;
+    const hasCoveragePlugin = plugins.some((p: {}) => 'reporter:coverage' in p);
+    const hasIstanbulPlugin = plugins.some((p: {}) => 'reporter:coverage-istanbul' in p);
+    const hasCoverageReporter = reporters.includes('coverage');
+    const hasIstanbulReporter = reporters.includes('coverage-istanbul');
+    if (hasCoveragePlugin && !hasCoverageReporter) {
+      reporters.push('coverage');
+    }
+    else if (hasIstanbulPlugin && !hasIstanbulReporter) {
+      // coverage-istanbul is deprecated in favor of karma-coverage
+      reporters.push('coverage-istanbul');
+    }
+    else {
+      throw new Error('karma-coverage must be installed in order to run code coverage');
+    }
+    if (hasIstanbulPlugin) {
+      logger.warn(`'karma-coverage-istanbul-reporter' usage has been deprecated since version 11.\n` +
+       `Please install 'karma-coverage' and update 'karma.conf.js.' ` +
+       'For more info, see https://github.com/karma-runner/karma-coverage/blob/master/README.md');
+    }
   }
-    
+
   // Add webpack config.
   const webpackConfig = config.buildWebpack.webpackConfig;
   const webpackMiddlewareConfig = {
@@ -188,7 +208,7 @@ const init: any = (config: any, emitter: any, customFileHandlers: any) => {
       logger.error(statsErrorsToString(json, statsConfig));
       lastCompilationHash = undefined;
       // Emit a failure build event if there are compilation errors.
-      failureCb && failureCb();
+      failureCb();
     } else if (stats.hash != lastCompilationHash) {
       // Refresh karma only when there are no webpack errors, and if the compilation changed.
       lastCompilationHash = stats.hash;
@@ -269,9 +289,9 @@ const eventReporter: any = function (this: any, baseReporterDecorator: any, conf
 
   this.onRunComplete = function (_browsers: any, results: any) {
     if (results.exitCode === 0) {
-      successCb && successCb();
+      successCb();
     } else {
-      failureCb && failureCb();
+      failureCb();
     }
   }
 

packages/angular_devkit/build_angular/src/karma/code-coverage_spec.ts

@@ -5,10 +5,24 @@
  * Use of this source code is governed by an MIT-style license that can be
  * found in the LICENSE file at https://angular.io/license
  */
+
+// tslint:disable:no-implicit-dependencies
+
 import { Architect } from '@angular-devkit/architect';
 import { normalize, virtualFs } from '@angular-devkit/core';
+import { last, tap } from 'rxjs/operators';
+import { promisify } from 'util';
 import { createArchitect, host, karmaTargetSpec } from '../test-utils';
 
+// In each of the test below we'll have to call setTimeout to wait for the coverage
+// analysis to be done. This is because karma-coverage performs the analysis
+// asynchronously but the promise that it returns is not awaited by Karma.
+// Coverage analysis begins when onRunComplete() is invoked, and output files
+// are subsequently written to disk. For more information, see
+// https://github.com/karma-runner/karma-coverage/blob/32acafa90ed621abd1df730edb44ae55a4009c2c/lib/reporter.js#L221
+
+const setTimeoutPromise = promisify(setTimeout);
+
 describe('Karma Builder code coverage', () => {
   const coverageFilePath = normalize('coverage/lcov.info');
   let architect: Architect;
@@ -23,16 +37,19 @@ describe('Karma Builder code coverage', () => {
   it('supports code coverage option', async () => {
     const run = await architect.scheduleTarget(karmaTargetSpec, { codeCoverage: true });
 
-    await expectAsync(run.result).toBeResolvedTo(jasmine.objectContaining({ success: true }));
+    const {success} = await run.result;
+    expect(success).toBe(true);
 
     await run.stop();
 
+    await setTimeoutPromise(1000);
+
     const exists = host.scopedSync().exists(coverageFilePath);
-    expect(exists).toBe(true);
+    expect(exists).toBe(true, `${coverageFilePath} does not exist`);
 
     if (exists) {
       const content = virtualFs.fileBufferToString(host.scopedSync().read(coverageFilePath));
-      expect(content).toContain('polyfills.ts');
+      expect(content).toContain('app.component.ts');
       expect(content).toContain('test.ts');
     }
   }, 120000);
@@ -41,23 +58,24 @@ describe('Karma Builder code coverage', () => {
     const overrides = {
       codeCoverage: true,
       codeCoverageExclude: [
-        'src/polyfills.ts',
         '**/test.ts',
       ],
     };
 
     const run = await architect.scheduleTarget(karmaTargetSpec, overrides);
 
-    await expectAsync(run.result).toBeResolvedTo(jasmine.objectContaining({ success: true }));
+    const {success} = await run.result;
+    expect(success).toBe(true);
 
     await run.stop();
 
+    await setTimeoutPromise(1000);
+
     const exists = host.scopedSync().exists(coverageFilePath);
     expect(exists).toBe(true);
 
     if (exists) {
       const content = virtualFs.fileBufferToString(host.scopedSync().read(coverageFilePath));
-      expect(content).not.toContain('polyfills.ts');
       expect(content).not.toContain('test.ts');
     }
   }, 120000);
@@ -98,10 +116,13 @@ describe('Karma Builder code coverage', () => {
 
     const run = await architect.scheduleTarget(karmaTargetSpec, { codeCoverage: true });
 
-    await expectAsync(run.result).toBeResolvedTo(jasmine.objectContaining({ success: true }));
+    const {success} = await run.result;
+    expect(success).toBe(true);
 
     await run.stop();
 
+    await setTimeoutPromise(1000);
+
     const exists = host.scopedSync().exists(coverageFilePath);
     expect(exists).toBe(true);
 
@@ -111,20 +132,18 @@ describe('Karma Builder code coverage', () => {
     }
   }, 120000);
 
-  it(`should fail when coverage is below threhold and 'emitWarning' is false`, async () => {
-    host.replaceInFile('karma.conf.js', 'fixWebpackSourcePaths: true',
-      `
-      fixWebpackSourcePaths: true,
-      thresholds: {
-        emitWarning: false,
-        global: {
-          statements: 100,
-          lines: 100,
-          branches: 100,
-          functions: 100
+  it('should exit with non-zero code when coverage is below threshold', async () => {
+    host.replaceInFile('karma.conf.js', 'coverageReporter: {', `
+      coverageReporter: {
+        check: {
+          global: {
+            statements: 100,
+            lines: 100,
+            branches: 100,
+            functions: 100
+          }
         },
-      }`,
-    );
+    `);
 
     host.appendToFile('src/app/app.component.ts', `
       export function nonCovered(): boolean {
@@ -133,7 +152,21 @@ describe('Karma Builder code coverage', () => {
     `);
 
     const run = await architect.scheduleTarget(karmaTargetSpec, { codeCoverage: true });
-    await expectAsync(run.result).toBeResolvedTo(jasmine.objectContaining({ success: false }));
+
+    // In incremental mode, karma-coverage does not have the ability to mark a
+    // run as failed if code coverage does not pass. This is because it does
+    // the coverage asynchoronously and Karma does not await the promise
+    // returned by the plugin.
+    expect((await run.result).success).toBeTrue();
+
+    // However the program must exit with non-zero exit code.
+    // This is a more common use case of coverage testing and must be supported.
+    await run.output.pipe(
+      last(),
+      tap(buildEvent => expect(buildEvent.success).toBeFalse()),
+    ).toPromise();
+
     await run.stop();
+
   }, 120000);
 });

packages/angular_devkit/build_angular/src/karma/index.ts

@@ -155,7 +155,10 @@ export function execute(
           // Complete the observable once the Karma server returns.
           const karmaServer = new karma.Server(
             transforms.karmaOptions ? transforms.karmaOptions(karmaOptions) : karmaOptions,
-            () => subscriber.complete(),
+            (exitCode: number) => {
+              subscriber.next({ success: exitCode === 0 });
+              subscriber.complete();
+            },
           );
           // karma typings incorrectly define start's return value as void
           // tslint:disable-next-line:no-use-of-empty-return-value

packages/angular_devkit/build_angular/test/hello-world-app/karma.conf.js

@@ -19,16 +19,19 @@ module.exports = function(config) {
       require('karma-jasmine'),
       require('karma-chrome-launcher'),
       require('karma-jasmine-html-reporter'),
-      require('karma-coverage-istanbul-reporter'),
+      require('karma-coverage'),
       require('@angular-devkit/build-angular/plugins/karma'),
     ],
     client: {
       clearContext: false, // leave Jasmine Spec Runner output visible in browser
     },
-    coverageIstanbulReporter: {
-      dir: path.join(__dirname, 'coverage'),
-      reports: ['html', 'lcovonly'],
-      fixWebpackSourcePaths: true,
+    coverageReporter: {
+      dir: path.join(__dirname, './coverage'),
+      subdir: '.',
+      reporters: [
+        {type: 'html'},
+        {type: 'lcov'},
+      ],
     },
     reporters: ['progress', 'kjhtml'],
     port: 9876,

packages/angular_devkit/build_ng_packagr/test/ng-packaged/projects/lib/karma.conf.js

@@ -16,16 +16,20 @@ module.exports = function (config) {
       require('karma-jasmine'),
       require('karma-chrome-launcher'),
       require('karma-jasmine-html-reporter'),
-      require('karma-coverage-istanbul-reporter'),
+      require('karma-coverage'),
       require('@angular-devkit/build-angular/plugins/karma')
     ],
     client: {
       clearContext: false // leave Jasmine Spec Runner output visible in browser
     },
-    coverageIstanbulReporter: {
+    coverageReporter: {
       dir: require('path').join(__dirname, 'coverage'),
-      reports: ['html', 'lcovonly'],
-      fixWebpackSourcePaths: true
+      subdir: '.',
+      reporters: [
+        {type: 'html'},
+        {type: 'lcov'},
+        {type: 'text-summary'},
+      ],
     },
     reporters: ['progress', 'kjhtml'],
     port: 9876,

packages/schematics/angular/application/files/karma.conf.js.template

@@ -9,16 +9,20 @@ module.exports = function (config) {
       require('karma-jasmine'),
       require('karma-chrome-launcher'),
       require('karma-jasmine-html-reporter'),
-      require('karma-coverage-istanbul-reporter'),
+      require('karma-coverage'),
       require('@angular-devkit/build-angular/plugins/karma')
     ],
     client: {
       clearContext: false // leave Jasmine Spec Runner output visible in browser
     },
-    coverageIstanbulReporter: {
+    coverageReporter: {
       dir: require('path').join(__dirname, '<%= relativePathToWorkspaceRoot %>/coverage/<%= appName%>'),
-      reports: ['html', 'lcovonly', 'text-summary'],
-      fixWebpackSourcePaths: true
+      subdir: '.',
+      reporters: [
+        {type: 'html'},
+        {type: 'lcov'},
+        {type: 'text-summary'},
+      ],
     },
     reporters: ['progress', 'kjhtml'],
     port: 9876,

packages/schematics/angular/library/files/karma.conf.js.template

@@ -9,16 +9,20 @@ module.exports = function (config) {
       require('karma-jasmine'),
       require('karma-chrome-launcher'),
       require('karma-jasmine-html-reporter'),
-      require('karma-coverage-istanbul-reporter'),
+      require('karma-coverage'),
       require('@angular-devkit/build-angular/plugins/karma')
     ],
     client: {
       clearContext: false // leave Jasmine Spec Runner output visible in browser
     },
-    coverageIstanbulReporter: {
+    coverageReporter: {
       dir: require('path').join(__dirname, '<%= relativePathToWorkspaceRoot %>/coverage/<%= folderName %>'),
-      reports: ['html', 'lcovonly', 'text-summary'],
-      fixWebpackSourcePaths: true
+      subdir: '.',
+      reporters: [
+        {type: 'html'},
+        {type: 'lcov'},
+        {type: 'text-summary'},
+      ],
     },
     reporters: ['progress', 'kjhtml'],
     port: 9876,

packages/schematics/angular/workspace/files/package.json.template

@@ -33,7 +33,7 @@
     "jasmine-spec-reporter": "~5.0.0",
     "karma": "~5.2.0",
     "karma-chrome-launcher": "~3.1.0",
-    "karma-coverage-istanbul-reporter": "~3.0.2",
+    "karma-coverage": "~2.0.3",
     "karma-jasmine": "~4.0.0",
     "karma-jasmine-html-reporter": "^1.5.0",
     "protractor": "~7.0.0",<% } %>