Revert 00ca06b; closes #3414 (#3715)

- Updates events test to match correct event order
- Add test for retries-and-bail case
- Split `--invert` test case into its own file

Author: boneskull

lib/runner.js

@@ -876,15 +876,11 @@ Runner.prototype.run = function(fn) {
     if (self._delay) {
       self.emit(constants.EVENT_DELAY_END);
     }
-    Runner.immediately(function() {
-      self.emit(constants.EVENT_RUN_BEGIN);
-    });
+    self.emit(constants.EVENT_RUN_BEGIN);
 
     self.runSuite(rootSuite, function() {
       debug('finished running');
-      Runner.immediately(function() {
-        self.emit(constants.EVENT_RUN_END);
-      });
+      self.emit(constants.EVENT_RUN_END);
     });
   }
 

test/assertions.js

@@ -100,6 +100,24 @@ exports.mixinMochaAssertions = function(expect) {
         code: expect.it('to be greater than', 0)
       });
     })
+    .addAssertion(
+      '<RawRunResult|RawResult> [not] to have failed (with|having) output <any>',
+      function(expect, result, output) {
+        expect(result, '[not] to satisfy', {
+          code: expect.it('to be greater than', 0),
+          output: output
+        });
+      }
+    )
+    .addAssertion(
+      '<RawRunResult|RawResult> [not] to have passed (with|having) output <any>',
+      function(expect, result, output) {
+        expect(result, '[not] to satisfy', {
+          code: 0,
+          output: output
+        });
+      }
+    )
     .addAssertion('<JSONRunResult> [not] to have test count <number>', function(
       expect,
       result,

test/integration/events.spec.js

@@ -75,4 +75,23 @@ describe('event order', function() {
       });
     });
   });
+
+  describe('--retries and --bail test case', function() {
+    it('should assert --retries event order', function(done) {
+      runMochaJSON(
+        'runner/events-bail-retries.fixture.js',
+        ['--retries', '1', '--bail'],
+        function(err, res) {
+          if (err) {
+            done(err);
+            return;
+          }
+          expect(res, 'to have failed with error', 'error test A')
+            .and('to have failed test count', 1)
+            .and('to have passed test count', 0);
+          done();
+        }
+      );
+    });
+  });
 });

test/integration/fixtures/runner/events-bail-retries.fixture.js

@@ -0,0 +1,57 @@
+'use strict';
+var Runner = require('../../../../lib/runner.js');
+var assert = require('assert');
+var constants = Runner.constants;
+var EVENT_HOOK_BEGIN = constants.EVENT_HOOK_BEGIN;
+var EVENT_HOOK_END = constants.EVENT_HOOK_END;
+var EVENT_RUN_BEGIN = constants.EVENT_RUN_BEGIN;
+var EVENT_RUN_END = constants.EVENT_RUN_END;
+var EVENT_SUITE_BEGIN = constants.EVENT_SUITE_BEGIN;
+var EVENT_SUITE_END = constants.EVENT_SUITE_END;
+var EVENT_TEST_BEGIN = constants.EVENT_TEST_BEGIN;
+var EVENT_TEST_END = constants.EVENT_TEST_END;
+var EVENT_TEST_FAIL = constants.EVENT_TEST_FAIL;
+var EVENT_TEST_RETRY = constants.EVENT_TEST_RETRY;
+
+var emitOrder = [
+ EVENT_RUN_BEGIN,
+  EVENT_SUITE_BEGIN,
+  EVENT_SUITE_BEGIN,
+  EVENT_HOOK_BEGIN,
+  EVENT_HOOK_END,
+  EVENT_TEST_BEGIN,
+  EVENT_HOOK_BEGIN,
+  EVENT_HOOK_END,
+  EVENT_TEST_RETRY,
+  EVENT_HOOK_BEGIN,
+  EVENT_HOOK_END,
+  EVENT_TEST_BEGIN,
+  EVENT_HOOK_BEGIN,
+  EVENT_HOOK_END,
+  EVENT_TEST_FAIL,
+  EVENT_TEST_END,
+  EVENT_HOOK_BEGIN,
+  EVENT_HOOK_END,
+  EVENT_HOOK_BEGIN,
+  EVENT_HOOK_END,
+  EVENT_SUITE_END,
+  EVENT_SUITE_END,
+  EVENT_RUN_END
+];
+
+var realEmit = Runner.prototype.emit;
+Runner.prototype.emit = function(event, ...args) {
+  // console.log(`emit: ${event}`);
+  assert.strictEqual(event, emitOrder.shift());
+  return realEmit.call(this, event, ...args);
+};
+
+describe('suite A', function() {
+  before('before', function() {});
+  beforeEach('beforeEach', function() {});
+  it('test A', function() {
+    throw new Error('error test A');
+  });
+  afterEach('afterEach', function() {});
+  after('after', function() {});
+});

test/integration/fixtures/runner/events-bail.fixture.js

@@ -13,9 +13,9 @@ var EVENT_TEST_END = constants.EVENT_TEST_END;
 var EVENT_TEST_FAIL = constants.EVENT_TEST_FAIL;
 
 var emitOrder = [
-  EVENT_SUITE_BEGIN, // incorrect order
   EVENT_RUN_BEGIN,
   EVENT_SUITE_BEGIN,
+  EVENT_SUITE_BEGIN,
   EVENT_HOOK_BEGIN,
   EVENT_HOOK_END,
   EVENT_TEST_BEGIN,

test/integration/fixtures/runner/events-basic.fixture.js

@@ -18,9 +18,9 @@ var EVENT_TEST_PENDING = constants.EVENT_TEST_PENDING;
 var EVENT_TEST_RETRY = constants.EVENT_TEST_RETRY;
 
 var emitOrder = [
-  EVENT_SUITE_BEGIN, // incorrect order
   EVENT_RUN_BEGIN,
   EVENT_SUITE_BEGIN,
+  EVENT_SUITE_BEGIN,
   EVENT_HOOK_BEGIN,
   EVENT_HOOK_END,
   EVENT_TEST_BEGIN,

test/integration/fixtures/runner/events-delay.fixture.js

@@ -20,9 +20,9 @@ var EVENT_TEST_RETRY = constants.EVENT_TEST_RETRY;
 var emitOrder = [
   EVENT_DELAY_BEGIN,
   EVENT_DELAY_END,
-  EVENT_SUITE_BEGIN, // incorrect order
   EVENT_RUN_BEGIN,
   EVENT_SUITE_BEGIN,
+  EVENT_SUITE_BEGIN,
   EVENT_HOOK_BEGIN,
   EVENT_HOOK_END,
   EVENT_TEST_BEGIN,

test/integration/fixtures/runner/events-retries.fixture.js

@@ -14,9 +14,9 @@ var EVENT_TEST_FAIL = constants.EVENT_TEST_FAIL;
 var EVENT_TEST_RETRY = constants.EVENT_TEST_RETRY;
 
 var emitOrder = [
-  EVENT_SUITE_BEGIN, // incorrect order
   EVENT_RUN_BEGIN,
   EVENT_SUITE_BEGIN,
+  EVENT_SUITE_BEGIN,
   EVENT_HOOK_BEGIN,
   EVENT_HOOK_END,
   EVENT_TEST_BEGIN,

test/integration/helpers.js

@@ -86,22 +86,29 @@ module.exports = {
     return invokeMocha(
       args,
       function(err, res) {
-        if (err) return fn(err);
+        if (err) {
+          return fn(err);
+        }
 
+        var result;
         try {
-          var result = toJSONRunResult(res);
-          fn(null, result);
+          // attempt to catch a JSON parsing error *only* here.
+          // previously, the callback was called within this `try` block,
+          // which would result in errors thrown from the callback
+          // getting caught by the `catch` block below.
+          result = toJSONRunResult(res);
         } catch (err) {
-          fn(
+          return fn(
             new Error(
               format(
-                'Failed to parse JSON reporter output. Error:\n%O\nResponse:\n%O',
+                'Failed to parse JSON reporter output. Error:\n%O\nResult:\n%O',
                 err,
                 res
               )
             )
           );
         }
+        fn(null, result);
       },
       opts
     );

test/integration/options/compilers.spec.js

@@ -4,20 +4,18 @@ var helpers = require('../helpers');
 var invokeMocha = helpers.invokeMocha;
 
 describe('--compilers', function() {
-  var args = [];
+  it('should report deprecation', function(done) {
+    invokeMocha(
+      ['--compilers', 'coffee:coffee-script/register'],
+      function(err, res) {
+        if (err) {
+          return done(err);
+        }
 
-  before(function() {
-    args = ['--compilers', 'coffee:coffee-script/register'];
-  });
-
-  it('should fail', function(done) {
-    invokeMocha(args, function(err, res) {
-      if (err) {
-        return done(err);
-      }
-
-      expect(res.code, 'to be', 1);
-      done();
-    });
+        expect(res, 'to have failed with output', /compilers is deprecated/i);
+        done();
+      },
+      'pipe'
+    );
   });
 });

test/integration/options/grep.spec.js

@@ -1,21 +1,14 @@
 'use strict';
 
-var path = require('path').posix;
 var helpers = require('../helpers');
 var runMocha = helpers.runMocha;
 var runMochaJSON = helpers.runMochaJSON;
 
-describe('--grep', function() {
-  var args = [];
-  var fixture = path.join('options', 'grep');
-
-  afterEach(function() {
-    args = [];
-  });
+var FIXTURE = 'options/grep';
 
+describe('--grep', function() {
   it('should run specs matching a string', function(done) {
-    args = ['--grep', 'match'];
-    runMochaJSON(fixture, args, function(err, res) {
+    runMochaJSON(FIXTURE, ['--grep', 'match'], function(err, res) {
       if (err) {
         return done(err);
       }
@@ -27,9 +20,8 @@ describe('--grep', function() {
   });
 
   describe('should run specs matching a RegExp', function() {
-    it('with RegExp-like strings (pattern follow by flag)', function(done) {
-      args = ['--grep', '/match/i'];
-      runMochaJSON(fixture, args, function(err, res) {
+    it('with RegExp-like strings (pattern followed by flag)', function(done) {
+      runMochaJSON(FIXTURE, ['--grep', '/match/i'], function(err, res) {
         if (err) {
           return done(err);
         }
@@ -41,8 +33,7 @@ describe('--grep', function() {
     });
 
     it('with string as pattern', function(done) {
-      args = ['--grep', '.*'];
-      runMochaJSON(fixture, args, function(err, res) {
+      runMochaJSON(FIXTURE, ['--grep', '.*'], function(err, res) {
         if (err) {
           return done(err);
         }
@@ -55,10 +46,9 @@ describe('--grep', function() {
     });
   });
 
-  describe('when --invert used', function() {
+  describe('when used with --invert', function() {
     it('should run specs that do not match the pattern', function(done) {
-      args = ['--grep', 'fail', '--invert'];
-      runMochaJSON(fixture, args, function(err, res) {
+      runMochaJSON(FIXTURE, ['--grep', 'fail', '--invert'], function(err, res) {
         if (err) {
           return done(err);
         }
@@ -68,40 +58,22 @@ describe('--grep', function() {
         done();
       });
     });
+  });
 
-    it('should throw an error when option used in isolation', function(done) {
-      var spawnOpts = {stdio: 'pipe'};
-      args = ['--invert'];
+  describe('when both --fgrep and --grep used together', function() {
+    it('should report an error', function(done) {
       runMocha(
-        fixture,
-        args,
+        FIXTURE,
+        ['--fgrep', 'first', '--grep', 'second'],
         function(err, res) {
           if (err) {
             return done(err);
           }
-          expect(res, 'to satisfy', {
-            code: 1,
-            output: /--invert.*--grep <regexp>/
-          });
+          expect(res, 'to have failed with output', /mutually exclusive/i);
           done();
         },
-        spawnOpts
+        'pipe'
       );
     });
   });
-
-  describe('when both --fgrep and --grep used together', function() {
-    it('should conflict', function(done) {
-      // var fixture = 'uncaught.fixture.js';
-      args = ['--fgrep', 'first', '--grep', 'second'];
-
-      runMocha(fixture, args, function(err, res) {
-        if (err) {
-          return done(err);
-        }
-        expect(res, 'to have failed');
-        done();
-      });
-    });
-  });
 });

test/integration/options/invert.spec.js

@@ -0,0 +1,26 @@
+'use strict';
+
+var runMocha = require('../helpers').runMocha;
+
+describe('--invert', function() {
+  describe('when used without --fgrep or --grep', function() {
+    it('it should report an error', function(done) {
+      runMocha(
+        'options/grep',
+        ['--invert'],
+        function(err, res) {
+          if (err) {
+            return done(err);
+          }
+          expect(
+            res,
+            'to have failed with output',
+            /--invert.*--grep <regexp>/
+          );
+          done();
+        },
+        'pipe'
+      );
+    });
+  });
+});