Arv document exported (#502)

* Add a flag to document all exported bindings

This adds a boolean flag called `document-exported` (defaults to false)
that effectively adds an empty comment to all exported bindings that do
not already have a JSDoc comment. It also does the same for the
members of exported classes and objects.

```js
export class C {
  method() {}
}
```

Both `C` and `C#method` are now part of the generated documentation.

Related to #424

* Fix lint error

* Rebase and use options pragma in test file

* Create extractor type as a generalized comment/export getter

* Document exported extractor

Author: tmcw

docs/USAGE.md

@@ -19,37 +19,41 @@ Commands:
   readme  inject documentation into your README.md
 
 Options:
-  --help             Show help                                         [boolean]
-  --version          Show version number                               [boolean]
-  --shallow          shallow mode turns off dependency resolution, only
-                     processing the specified files (or the main script
-                     specified in package.json)       [boolean] [default: false]
-  --config, -c       configuration file. an array defining explicit sort order
-  --external         a string / glob match pattern that defines which external
-                     modules will be whitelisted and included in the generated
-                     documentation.                              [default: null]
-  --extension, -e    only input source files matching this extension will be
-                     parsed, this option can be used multiple times.
-  --polyglot         polyglot mode turns off dependency resolution and enables
-                     multi-language support. use this to document c++  [boolean]
-  --private, -p      generate documentation tagged as private
+  --help               Show help                                       [boolean]
+  --version            Show version number                             [boolean]
+  --shallow            shallow mode turns off dependency resolution, only
+                       processing the specified files (or the main script
+                       specified in package.json)     [boolean] [default: false]
+  --config, -c         configuration file. an array defining explicit sort order
+  --external           a string / glob match pattern that defines which external
+                       modules will be whitelisted and included in the generated
+                       documentation.                            [default: null]
+  --extension, -e      only input source files matching this extension will be
+                       parsed, this option can be used multiple times.
+  --polyglot           polyglot mode turns off dependency resolution and enables
+                       multi-language support. use this to document c++[boolean]
+  --private, -p        generate documentation tagged as private
                                                       [boolean] [default: false]
-  --access, -a       Include only comments with a given access level, out of
-                     private, protected, public, undefined. By default, public,
-                     protected, and undefined access levels are included
+  --access, -a         Include only comments with a given access level, out of
+                       private, protected, public, undefined. By default,
+                       public, protected, and undefined access levels are
+                       included
                         [choices: "public", "private", "protected", "undefined"]
-  --github, -g       infer links to github in documentation            [boolean]
-  --infer-private    Infer private access based on the name. This is a regular
-                     expression that is used to match the name          [string]
-  --theme, -t        specify a theme: this must be a valid theme module
-  --name             project name. by default, inferred from package.json
-  --watch, -w        watch input files and rebuild documentation when they
-                     change                                            [boolean]
-  --project-version  project version. by default, inferred from package.json
-  --output, -o       output location. omit for stdout, otherwise is a filename
-                     for single-file outputs and a directory name for multi-file
-                     outputs like html                       [default: "stdout"]
-  --format, -f                 [choices: "json", "md", "html"] [default: "json"]
+  --github, -g         infer links to github in documentation          [boolean]
+  --infer-private      Infer private access based on the name. This is a regular
+                       expression that is used to match the name        [string]
+  --document-exported  Generate documentation for all exported bindings and
+                       members even if there is no JSDoc for them
+                                                      [boolean] [default: false]
+  --theme, -t          specify a theme: this must be a valid theme module
+  --name               project name. by default, inferred from package.json
+  --watch, -w          watch input files and rebuild documentation when they
+                       change                                          [boolean]
+  --project-version    project version. by default, inferred from package.json
+  --output, -o         output location. omit for stdout, otherwise is a filename
+                       for single-file outputs and a directory name for
+                       multi-file outputs like html          [default: "stdout"]
+  --format, -f       [choices: "json", "md", "remark", "html"] [default: "json"]
 
 Examples:
   documentation build foo.js -f md >        parse documentation in a file and

index.js

@@ -194,7 +194,7 @@ function buildSync(indexes, options) {
             return [];
           }
 
-          return parseFn(indexObject).map(buildPipeline);
+          return parseFn(indexObject, options).map(buildPipeline);
         })
         .filter(Boolean), options)));
 }
@@ -249,7 +249,7 @@ function lint(indexes, options, callback) {
         inputs
           .filter(filterJS(options.extension, options.polyglot))
           .reduce(function (memo, file) {
-            return memo.concat(parseFn(file).map(lintPipeline));
+            return memo.concat(parseFn(file, options).map(lintPipeline));
           }, [])
           .filter(Boolean))));
   });

lib/commands/shared_options.js

@@ -53,6 +53,12 @@ function sharedInputOptions(parser) {
     type: 'string',
     describe: 'Infer private access based on the name. This is a regular expression that ' +
       'is used to match the name'
+  })
+  .option('document-exported', {
+    type: 'boolean',
+    describe: 'Generate documentation for all exported bindings and members ' +
+      'even if there is no JSDoc for them',
+    default: false
   });
 }
 

lib/extractors/comments.js

@@ -0,0 +1,47 @@
+var traverse = require('babel-traverse').default,
+  isJSDocComment = require('../../lib/is_jsdoc_comment');
+
+/**
+ * Iterate through the abstract syntax tree, finding a different kind of comment
+ * each time, and optionally including context. This is how we find
+ * JSDoc annotations that will become part of documentation
+ * @param {string} type comment type to find
+ * @param {boolean} includeContext to include context in the nodes
+ * @param {Object} ast the babel-parsed syntax tree
+ * @param {Function} addComment a method that creates a new comment if necessary
+ * @returns {Array<Object>} comments
+ * @private
+ */
+function walkComments(type, includeContext, ast, addComment) {
+  var newResults = [];
+
+  traverse(ast, {
+    /**
+     * Process a parse in an abstract syntax tree
+     * @param {Object} path ast path
+     * @returns {undefined} causes side effects
+     * @private
+     */
+    enter: function (path) {
+      /**
+       * Parse a comment with doctrine and decorate the result with file position and code context.
+       *
+       * @param {Object} comment the current state of the parsed JSDoc comment
+       * @return {undefined} this emits data
+       */
+      function parseComment(comment) {
+        newResults.push(addComment(comment.value, comment.loc, path, path.node.loc, includeContext));
+      }
+
+      (path.node[type] || [])
+        .filter(isJSDocComment)
+        .forEach(parseComment);
+    }
+  });
+
+  traverse.clearCache();
+
+  return newResults;
+}
+
+module.exports = walkComments;

lib/extractors/exported.js

@@ -0,0 +1,63 @@
+var traverse = require('babel-traverse').default,
+  isJSDocComment = require('../../lib/is_jsdoc_comment');
+
+
+/**
+ * Iterate through the abstract syntax tree, finding ES6-style exports,
+ * and inserting blank comments into documentation.js's processing stream.
+ * Through inference steps, these comments gain more information and are automatically
+ * documented as well as we can.
+ * @param {Object} ast the babel-parsed syntax tree
+ * @param {Function} addComment a method that creates a new comment if necessary
+ * @returns {Array<Object>} comments
+ * @private
+ */
+function walkExported(ast, addComment) {
+  var newResults = [];
+
+  function addBlankComment(path, node) {
+    return addComment('', node.loc, path, node.loc, true);
+  }
+
+  traverse(ast, {
+    enter: function (path) {
+      if (path.isExportDeclaration()) {
+        if (!hasJSDocComment(path)) {
+          if (!path.node.declaration) {
+            return;
+          }
+          const node = path.node.declaration;
+          newResults.push(addBlankComment(path, node));
+        }
+      } else if ((path.isClassProperty() || path.isClassMethod()) &&
+          !hasJSDocComment(path) && inExportedClass(path)) {
+        newResults.push(addBlankComment(path, path.node));
+      } else if ((path.isObjectProperty() || path.isObjectMethod()) &&
+          !hasJSDocComment(path) && inExportedObject(path)) {
+        newResults.push(addBlankComment(path, path.node));
+      }
+    }
+  });
+
+  return newResults;
+}
+
+function hasJSDocComment(path) {
+  return path.node.leadingComments && path.node.leadingComments.some(isJSDocComment);
+}
+
+function inExportedClass(path) {
+  var c = path.parentPath.parentPath;
+  return c.isClass() && c.parentPath.isExportDeclaration();
+}
+
+function inExportedObject(path) {
+  // ObjectExpression -> VariableDeclarator -> VariableDeclaration -> ExportNamedDeclaration
+  var p = path.parentPath.parentPath;
+  if (!p.isVariableDeclarator()) {
+    return false;
+  }
+  return p.parentPath.parentPath.isExportDeclaration();
+}
+
+module.exports = walkExported;

lib/parsers/javascript.js

@@ -1,10 +1,11 @@
 'use strict';
 
 var babylon = require('babylon'),
-  traverse = require('babel-traverse').default,
   extend = require('extend'),
-  isJSDocComment = require('../../lib/is_jsdoc_comment'),
-  parse = require('../../lib/parse');
+  _ = require('lodash'),
+  parse = require('../../lib/parse'),
+  walkComments = require('../extractors/comments'),
+  walkExported = require('../extractors/exported');
 
 /**
  * Left-pad a string so that it can be sorted lexicographically. We sort
@@ -27,10 +28,13 @@ function leftPad(str, width) {
  * reads the file, parses the JavaScript, and parses the JSDoc.
  *
  * @param {Object} data a chunk of data provided by module-deps
+ * @param {Object} options options
  * @return {Array<Object>} an array of parsed comments
  */
-function parseJavaScript(data) {
-  var results = [];
+function parseJavaScript(data, options) {
+  options = options || {};
+  var visited = {};
+
   var ast = babylon.parse(data.source, {
     allowImportExportEverywhere: true,
     sourceType: 'module',
@@ -52,75 +56,44 @@ function parseJavaScript(data) {
     ]
   });
 
-  var visited = {};
+  var addComment = _addComment.bind(null, visited, data);
 
-  /**
-   * Iterate through the abstract syntax tree, finding a different kind of comment
-   * each time, and optionally including context. This is how we find
-   * JSDoc annotations that will become part of documentation
-   * @param {Object} ast the babel-parsed syntax tree
-   * @param {string} type comment type to find
-   * @param {boolean} includeContext to include context in the nodes
-   * @returns {Array<Object>} comments
-   * @private
-   */
-  function walkComments(ast, type, includeContext) {
-    traverse(ast, {
-      /**
-       * Process a parse in an abstract syntax tree
-       * @param {Object} path ast path
-       * @returns {undefined} causes side effects
-       * @private
-       */
-      enter: function (path) {
-        /**
-         * Parse a comment with doctrine and decorate the result with file position and code context.
-         *
-         * @param {Object} comment the current state of the parsed JSDoc comment
-         * @return {undefined} this emits data
-         */
-        function parseComment(comment) {
-          var context = {
-            loc: extend({}, JSON.parse(JSON.stringify(path.node.loc))),
-            file: data.file,
-            sortKey: data.sortKey + ' ' + leftPad(path.node.loc.start.line, 8)
-          };
-          // Avoid visiting the same comment twice as a leading
-          // and trailing node
-          var key = JSON.stringify(comment.loc);
-          if (!visited[key]) {
-            visited[key] = true;
-            if (includeContext) {
-              // This is non-enumerable so that it doesn't get stringified in
-              // output; e.g. by the documentation binary.
-              Object.defineProperty(context, 'ast', {
-                enumerable: false,
-                value: path
-              });
+  return _.flatMap([
+    walkComments.bind(null, 'leadingComments', true),
+    walkComments.bind(null, 'innerComments', false),
+    walkComments.bind(null, 'trailingComments', false),
+    options.documentExported && walkExported
+  ].filter(Boolean), function (fn) {
+    return fn(ast, addComment);
+  }).filter(Boolean);
+}
 
-              if (path.parentPath && path.parentPath.node) {
-                context.code = data.source.substring
-                  .apply(data.source, path.parentPath.node.range);
-              }
-            }
-            results.push(parse(comment.value, comment.loc, context));
-          }
-        }
+function _addComment(visited, data, commentValue, commentLoc, path, nodeLoc, includeContext) {
+  var context = {
+    loc: extend({}, JSON.parse(JSON.stringify(nodeLoc))),
+    file: data.file,
+    sortKey: data.sortKey + ' ' + leftPad(nodeLoc.start.line, 8)
+  };
+  // Avoid visiting the same comment twice as a leading
+  // and trailing node
+  var key = JSON.stringify(commentLoc);
+  if (!visited[key]) {
+    visited[key] = true;
+    if (includeContext) {
+      // This is non-enumerable so that it doesn't get stringified in
+      // output; e.g. by the documentation binary.
+      Object.defineProperty(context, 'ast', {
+        enumerable: false,
+        value: path
+      });
 
-        (path.node[type] || [])
-          .filter(isJSDocComment)
-          .forEach(parseComment);
+      if (path.parentPath && path.parentPath.node) {
+        context.code = data.source.substring
+          .apply(data.source, path.parentPath.node.range);
       }
-    });
-
-    traverse.clearCache();
+    }
+    return parse(commentValue, commentLoc, context);
   }
-
-  walkComments(ast, 'leadingComments', true);
-  walkComments(ast, 'innerComments', false);
-  walkComments(ast, 'trailingComments', false);
-
-  return results;
 }
 
 module.exports = parseJavaScript;

test/bin.js

@@ -341,3 +341,19 @@ test('fatal error', options, function (t) {
       t.end();
     }, false);
 });
+
+test('build --document-exported', function (t) {
+
+  documentation(['build fixture/document-exported.input.js --document-exported -f md'], {}, function (err, data) {
+    t.error(err);
+
+    var outputfile = path.join(__dirname, 'fixture', 'document-exported.output.md');
+    if (process.env.UPDATE) {
+      fs.writeFileSync(outputfile, data, 'utf8');
+    }
+
+    var expect = fs.readFileSync(outputfile, 'utf-8');
+    t.equal(data, expect);
+    t.end();
+  }, false);
+}, options);