New: consistent-output rule

Author: not-an-aardvark

README.md

@@ -55,6 +55,7 @@ Name | ✔️ | 🛠 | Description
 [test-case-shorthand-strings](https://github.com/not-an-aardvark/eslint-plugin-eslint-plugin/blob/master/docs/rules/test-case-shorthand-strings.md) | | | Enforces consistent usage of shorthand strings for test cases with no options
 [prefer-placeholders](https://github.com/not-an-aardvark/eslint-plugin-eslint-plugin/blob/master/docs/rules/prefer-placeholders.md) | | | Disallows template literals as report messages
 [no-useless-token-range](https://github.com/not-an-aardvark/eslint-plugin-eslint-plugin/blob/master/docs/rules/no-useless-token-range.md) | ✔️ | 🛠 | Disallows unnecessary calls to sourceCode.getFirstToken and sourceCode.getLastToken
+[consistent-output](https://github.com/not-an-aardvark/eslint-plugin-eslint-plugin/blob/master/docs/rules/consistent-output.md) | | | Enforces consistent use of output assertions in rule tests
 
 ## Supported Presets
 

docs/rules/consistent-output.md

@@ -0,0 +1,60 @@
+# Enforce consistent use of output assertions in rule tests (consistent-output)
+
+When writing tests for a fixable rule with `RuleTester`, you can assert the autofix output of your test cases. However, it can be easy to forget to assert the output of a particular test case.
+
+## Rule Details
+
+This rule aims to ensure that if any invalid test cases have output assertions, then all test cases have output assertions.
+
+Examples of **incorrect** code for this rule:
+
+```js
+/* eslint eslint-plugin/consistent-output: error */
+
+new RuleTester().run('example-rule', rule, {
+  valid: [],
+  invalid: [
+    {
+      code: 'foo',
+      output: 'bar',
+      errors: ['baz']
+    },
+    {
+      code: 'bar',
+      errors: ['baz']
+    }
+  ]
+});
+
+```
+
+Examples of **correct** code for this rule:
+
+```js
+/* eslint eslint-plugin/consistent-output: error */
+
+new RuleTester().run('example-rule', rule, {
+  valid: [],
+  invalid: [
+    {
+      code: 'foo',
+      output: 'bar',
+      errors: ['baz']
+    },
+    {
+      code: 'bar',
+      output: 'qux',
+      errors: ['baz']
+    }
+  ]
+});
+
+```
+
+## When Not To Use It
+
+If you're not writing fixable rules, or you want to write test cases without output assertions, do not enable this rule.
+
+## Further Reading
+
+* [`RuleTester` documentation](http://eslint.org/docs/developer-guide/working-with-plugins#testing)

lib/rules/consistent-output.js

@@ -0,0 +1,52 @@
+/**
+ * @fileoverview Enforce consistent use of output assertions in rule tests
+ * @author Teddy Katz
+ */
+
+'use strict';
+
+const utils = require('../utils');
+
+// ------------------------------------------------------------------------------
+// Rule Definition
+// ------------------------------------------------------------------------------
+
+module.exports = {
+  meta: {
+    docs: {
+      description: 'Enforce consistent use of output assertions in rule tests',
+      category: 'Fill me in',
+      recommended: false,
+    },
+    fixable: null,  // or "code" or "whitespace"
+    schema: [],
+  },
+
+  create (context) {
+    // ----------------------------------------------------------------------
+    // Public
+    // ----------------------------------------------------------------------
+
+    return {
+      Program (ast) {
+        utils.getTestInfo(context, ast).forEach(testRun => {
+          if (!testRun.invalid) {
+            return;
+          }
+          const readableCases = testRun.invalid.filter(testCase => testCase.type === 'ObjectExpression');
+          const casesWithoutOutput = readableCases
+            .filter(testCase => testCase.properties.map(utils.getKeyName).indexOf('output') === -1);
+
+          if (casesWithoutOutput.length < readableCases.length) {
+            casesWithoutOutput.forEach(testCase => {
+              context.report({
+                node: testCase,
+                message: 'This test case should have an output assertion.',
+              });
+            });
+          }
+        });
+      },
+    };
+  },
+};

tests/lib/rules/consistent-output.js

@@ -0,0 +1,83 @@
+/**
+ * @fileoverview Enforce consistent use of output assertions in rule tests
+ * @author Teddy Katz
+ */
+
+'use strict';
+
+// ------------------------------------------------------------------------------
+// Requirements
+// ------------------------------------------------------------------------------
+
+const rule = require('../../../lib/rules/consistent-output');
+const RuleTester = require('eslint').RuleTester;
+
+const ERROR = { message: 'This test case should have an output assertion.', type: 'ObjectExpression' };
+
+// ------------------------------------------------------------------------------
+// Tests
+// ------------------------------------------------------------------------------
+
+const ruleTester = new RuleTester();
+ruleTester.run('consistent-output', rule, {
+  valid: [
+    `
+      new RuleTester().run('foo', bar, {
+        valid: [],
+        invalid: [
+          {
+            code: 'foo',
+            errors: ['bar']
+          },
+          {
+            code: 'baz',
+            errors: ['qux']
+          }
+        ]
+      });
+    `,
+    `
+      new RuleTester().run('foo', bar, {
+        valid: [],
+        invalid: [
+          {
+            code: 'foo',
+            output: 'baz',
+            errors: ['bar'],
+          },
+          {
+            code: 'foo',
+            output: 'qux',
+            errors: ['bar']
+          }
+        ]
+      });
+    `,
+  ],
+
+  invalid: [
+    {
+      code: `
+        new RuleTester().run('foo', bar, {
+          valid: [],
+          invalid: [
+            {
+              code: 'foo',
+              output: 'baz',
+              errors: ['bar'],
+            },
+            {
+              code: 'foo',
+              errors: ['bar']
+            },
+            {
+              code: 'foo bar',
+              errors: ['bar']
+            }
+          ]
+        });
+      `,
+      errors: [ERROR, ERROR],
+    },
+  ],
+});