docs(api/dart): add support for generation and display

Fixes #1880.
Supersedes #1593.
### Task breakdown

**Package dependencies:**
- [ ] [**BLOCKING**] Update to new [harp](https://github.com/sintaxi/harp) package which depends on latest [terraform](https://github.com/sintaxi/terraform) release. This is needed so that dots (`.`) can be supported in site URIs; something that is essential for Dart given that library names contain dots. See sintaxi/harp#526 (/cc @sintaxi).<br>Blocked awaiting the new release.

**Gulp tasks / build scripts:**
- [x] New tasks to run `pub` and `dartdoc` over (existing) `angular-dart` repo, peer to ng.io repo.
- [x] `build-api-docs` should also do Dart API docs.
- [x] Shredder should run over Dart sources and generate example fragments.
- [x] Create script to patch local `terraform` package, while we await new harp release, as a means of **unblocking**.

**Generator package:**
- [x] New `tools/dart-api-builder`; stand alone tool re-using part of the basis of a Dart API dgeni package.

**Harp/Jade site infrastructure:**
- [x] New `public/docs/_layout-dart-api.jade`.
- [x] Updates to other Harp/Jade template files.

**API List directive:**
- [x] Create new or update existing `<api-list>` directive:
  - [x] Display only filter categories that are relevant to Dart.

**Site data and Jade file generation:**
- [x] Main API page
  - [x] Generate `api-list.json`.
  - [x] Add warning that API is preliminary and that code might still appear as TS.
- [x] Generate data (`_data.json`) and Jade files for
  - [x] For each library (i.e., top-level folder).
    - [x] `_data.json`
    - [x] Library "index" Jade page.
    - [x] Each member Jade page.
  - [x] For library subfolders (e.g., containing class member details).
    - [x] `_data.json`
    - [x] Folder "index" page (it actually has the same name as the container -- e.g., class name).
    - [x] Each member page.
- [x] API Jade file page contents:
  - [x] Attributes like name and type
  - [x] Breadcrumbs
  - [ ] Page header section tabs.
  - [x] Core page content excerpt from dartdoc-generated page
    - [x] Exclude the "left" nav menu (full ng2 package content list) from the core page content.
  - [x] `{@example ...}` tags are replaced by their designated example code fragments.
- [x] Filter generated API pages (configurable via regExp in build file): exclude entries from
  - [x] Non-public, developer/internal libraries.
  - [x] Top-level catch all `angular2` library -- otherwise every entry appears twice.
- [x] Support developer & production modes for data (JSON) file generation
  - [x] Developer mode (pretty-printed).
  - [x] [NOT implemented] ~~Production mode (compact).~~

Note about the API subsite design:
- I had to resort to using Jade extends/includes; hence disabling Harp _layout/partials.

cc: @naomiblack @kwalrath @ferhatb @keertip

Author: chalin

gulpfile.js

@@ -30,6 +30,7 @@ var tslint = require('gulp-tslint');
 //  2. Think about using spawn instead of exec in case of long error messages.
 
 var TOOLS_PATH = './tools';
+var ANGULAR_IO_PROJECT_PATH = path.resolve('.');
 var ANGULAR_PROJECT_PATH = '../angular';
 var PUBLIC_PATH = './public';
 var TEMP_PATH = './_temp';
@@ -63,12 +64,21 @@ var _devguideShredJadeOptions =  {
 };
 
 var _apiShredOptions =  {
+  lang: 'ts',
   examplesDir: path.join(ANGULAR_PROJECT_PATH, 'modules/@angular/examples'),
   fragmentsDir: path.join(DOCS_PATH, '_fragments/_api'),
   zipDir: path.join(RESOURCES_PATH, 'zips/api'),
   logLevel: _dgeniLogLevel
 };
 
+var _apiShredOptionsForDart =  {
+  lang: 'dart',
+  examplesDir: path.resolve(ngPathFor('dart'), 'examples'),
+  fragmentsDir: path.join(DOCS_PATH, '_fragments/_api'),
+  zipDir: path.join(RESOURCES_PATH, 'zips/api'),
+  logLevel: _dgeniLogLevel
+};
+
 var _excludePatterns = ['**/node_modules/**', '**/typings/**', '**/packages/**'];
 
 var _excludeMatchers = _excludePatterns.map(function(excludePattern){
@@ -96,6 +106,14 @@ var _exampleProtractorBoilerplateFiles = [
 
 var _exampleConfigFilename = 'example-config.json';
 
+var lang, langs;
+function configLangs(langOption) {
+  lang = (langOption || 'all').toLowerCase();
+  if (lang === 'all') { lang = '(ts|js|dart)'; }
+  langs = lang.match(/\w+/g); // the languages in `lang` as an array
+}
+configLangs(argv.lang);
+
 function isDartPath(path) {
   // Testing via indexOf() for now. If we need to match only paths with folders
   // named 'dart' vs 'dart*' then try: path.match('/dart(/|$)') != null;
@@ -131,6 +149,7 @@ gulp.task('run-e2e-tests', runE2e);
  *     all means (ts|js|dart)
  */
 function runE2e() {
+  if (!argv.lang) configLangs('ts|js'); // Exclude dart by default
   var promise;
   if (argv.fast) {
     // fast; skip all setup
@@ -183,8 +202,6 @@ function runE2e() {
 // each app/spec collection sequentially.
 function findAndRunE2eTests(filter, outputFile) {
   // create an output file with header.
-  var lang = (argv.lang || '(ts|js)').toLowerCase();
-  if (lang === 'all') { lang = '(ts|js|dart)'; }
   var startTime = new Date().getTime();
   var header = `Doc Sample Protractor Results for ${lang} on ${new Date().toLocaleString()}\n`;
   header += argv.fast ?
@@ -496,7 +513,7 @@ gulp.task('build-docs', ['build-devguide-docs', 'build-api-docs', 'build-plunker
 // Stop zipping examples Feb 28, 2016
 //gulp.task('build-docs', ['build-devguide-docs', 'build-api-docs', 'build-plunkers', '_zip-examples']);
 
-gulp.task('build-api-docs', ['build-js-api-docs', 'build-ts-api-docs', 'build-dart-cheatsheet']);
+gulp.task('build-api-docs', ['build-js-api-docs', 'build-ts-api-docs', 'build-dart-api-docs']);
 
 gulp.task('build-devguide-docs', ['_shred-devguide-examples', '_shred-devguide-shared-jade'], function() {
   return buildShredMaps(true);
@@ -510,12 +527,40 @@ gulp.task('build-js-api-docs', ['_shred-api-examples'], function() {
   return buildApiDocs('js');
 });
 
+gulp.task('build-dart-api-docs', ['_shred-api-examples', 'dartdoc'], function() {
+  // TODO(chalin): also build build-dart-cheatsheet
+  return buildApiDocsForDart();
+});
+
 gulp.task('build-plunkers', ['_copy-example-boilerplate'], function() {
   return plunkerBuilder.buildPlunkers(EXAMPLES_PATH, LIVE_EXAMPLES_PATH, { errFn: gutil.log });
 });
 
 gulp.task('build-dart-cheatsheet', [], function() {
-  return buildApiDocs('dart');
+  gutil.log('build-dart-cheatsheet - NOT IMPLEMENTED YET');
+  // return buildApiDocsForDart();
+});
+
+gulp.task('dartdoc', ['pub upgrade'], function() {
+  const ngRepoPath = ngPathFor('dart');
+  if (argv.fast && fs.existsSync(path.resolve(ngRepoPath, 'doc'))) {
+    gutil.log('Skipping dartdoc: --fast flag enabled and "doc" dir exists');
+    return true;
+  }
+  checkAngularProjectPath(ngRepoPath);
+  const dartdoc = spawnExt('dartdoc', ['--output', 'doc/api', '--add-crossdart'], { cwd: ngRepoPath});
+  return dartdoc.promise;
+});
+
+gulp.task('pub upgrade', [], function() {
+  const ngRepoPath = ngPathFor('dart');
+  if (argv.fast && fs.existsSync(path.resolve(ngRepoPath, 'packages'))) {
+    gutil.log('Skipping pub upgrade: --fast flag enabled and "packages" dir exists');
+    return true;
+  }
+  checkAngularProjectPath(ngRepoPath);
+  const pubUpgrade = spawnExt('pub', ['upgrade'], { cwd: ngRepoPath});
+  return pubUpgrade.promise;
 });
 
 gulp.task('git-changed-examples', ['_shred-devguide-examples'], function(){
@@ -564,6 +609,35 @@ gulp.task('git-changed-examples', ['_shred-devguide-examples'], function(){
   });
 });
 
+gulp.task('harp-compile', [], function() {
+  return harpCompile()
+});
+
+gulp.task('serve', [], function() {
+  // Harp will serve files from workspace.
+  const cmd = 'npm run harp -- server .';
+  gutil.log('Launching harp server (over project files)');
+  gutil.log(`  > ${cmd}`);
+  gutil.log('Note: issuing this command directly from the command line will show harp comiple warnings.');
+  return execPromise(cmd);
+});
+
+gulp.task('serve-www', [], function() {
+  // Serve generated site.
+  return execPromise('npm run live-server ./www');
+});
+
+gulp.task('check', ['build-docs'], function() {
+  return harpCompile();
+});
+
+gulp.task('check-serve', ['build-docs'], function() {
+  return harpCompile().then(function() {
+    gutil.log('Launching live-server over ./www');
+    return execPromise('npm run live-server ./www');
+  });
+});
+
 gulp.task('check-deploy', ['build-docs'], function() {
   return harpCompile().then(function() {
     gutil.log('compile ok');
@@ -661,8 +735,15 @@ gulp.task('_shred-clean-devguide', function(cb) {
 });
 
 gulp.task('_shred-api-examples', ['_shred-clean-api'], function() {
-  checkAngularProjectPath();
-  return docShredder.shred(_apiShredOptions);
+  const promises = [];
+  gutil.log('Shredding API examples for languages: ' + langs.join(', '));
+  langs.forEach((lang) => {
+    if (lang === 'js') return; // JS is handled via TS.
+    checkAngularProjectPath(ngPathFor(lang));
+    const options = lang == 'dart' ? _apiShredOptionsForDart : _apiShredOptions;
+    promises.push(docShredder.shred(options));
+  });
+  return Q.all(promises);
 });
 
 gulp.task('_shred-clean-api', function(cb) {
@@ -1053,8 +1134,8 @@ function buildApiDocs(targetLanguage) {
     var dgeni = new Dgeni([package]);
     return dgeni.generate();
   } catch(err) {
-    gutil.log(err);
-    gutil.log(err.stack);
+    console.error(err);
+    console.error(err.stack);
     throw err;
   }
 
@@ -1065,6 +1146,48 @@ function buildApiDocs(targetLanguage) {
   }
 }
 
+
+function buildApiDocsForDart() {
+  const apiDir = 'api';
+  const vers = 'latest';
+  const dab = require('./tools/dart-api-builder/dab')(ANGULAR_IO_PROJECT_PATH);
+  const log = dab.log;
+
+  log.level = _dgeniLogLevel;
+  const dabInfo = dab.dartPkgConfigInfo;
+  dabInfo.ngIoDartApiDocPath = path.join(DOCS_PATH, 'dart', vers, apiDir);
+  dabInfo.ngDartDocPath = path.join(ngPathFor('dart'), 'doc', apiDir);
+  // Exclude API entries for developer/internal libraries. Also exclude entries for
+  // the top-level catch all "angular2" library (otherwise every entry appears twice).
+  dabInfo.excludeLibRegExp = new RegExp(/^(?!angular2)|\.testing|_|codegen|^angular2$/);
+
+  try {
+    checkAngularProjectPath('dart');
+    var destPath = dabInfo.ngIoDartApiDocPath;
+    var sourceDirs = fs.readdirSync(dabInfo.ngDartDocPath)
+      .filter((name) => !name.match(/^index/))
+      .map((name) => path.join(dabInfo.ngDartDocPath, name));
+    log.info(`Building Dart API pages for ${sourceDirs.length} libraries`);
+
+    return copyFiles(sourceDirs, [destPath]).then(() => {
+      log.debug('Finished copying', sourceDirs.length, 'directories from', dabInfo.ngDartDocPath, 'to', destPath);
+
+      const apiEntries = dab.loadApiDataAndSaveToApiListFile();
+      const tmpDocsPath = path.resolve(path.join(process.env.HOME, 'tmp/docs.json'));
+      if (argv.dumpDocsJson) fs.writeFileSync(tmpDocsPath, JSON.stringify(apiEntries, null, 2));
+      dab.createApiDataAndJadeFiles(apiEntries);
+
+    }).catch((err) => {
+      console.log(err);    
+    });
+
+  } catch(err) {
+    console.error(err);
+    console.error(err.stack);
+    throw err;
+  }
+}
+
 function buildShredMaps(shouldWrite) {
   var options = {
     devguideExamplesDir: _devguideShredOptions.examplesDir,
@@ -1236,8 +1359,13 @@ function execCommands(cmds, options, cb) {
   });
 }
 
-function checkAngularProjectPath() {
-  if (!fs.existsSync(ANGULAR_PROJECT_PATH)) {
-    throw new Error('API related tasks require the angular2 repo to be at ' + path.resolve(ANGULAR_PROJECT_PATH));
+function ngPathFor(lang) {
+  return ANGULAR_PROJECT_PATH + (lang === 'dart' ? '-dart' : '');
+}
+
+function checkAngularProjectPath(lang) {
+  var ngPath = path.resolve(ngPathFor(lang || 'ts'));
+  if (!fs.existsSync(ngPath)) {
+    throw new Error('API related tasks require the angular2 repo to be at ' + ngPath);
   }
 }

package.json

@@ -30,6 +30,7 @@
     "broken-link-checker": "0.7.1",
     "browser-sync": "^2.9.3",
     "canonical-path": "0.0.2",
+    "cheerio": "^0.20.0",
     "cross-spawn": "^4.0.0",
     "codelyzer": "0.0.22",
     "del": "^2.2.0",

public/_includes/_hero.jade

@@ -1,19 +1,19 @@
-// Refer to jade.template.html and addJadeDataDocsProcessor to figure out where the context of this jade file originates
+// template: public/_includes/_hero
+//- Refer to jade.template.html and addJadeDataDocsProcessor to figure out where the context of this jade file originates
 - var textFormat = '';
 - var headerTitle = title + (typeof varType !== 'undefined' ? (': ' + varType) : '');
 - var capitalize = function capitalize(str) { return str.charAt(0).toUpperCase() + str.slice(1); }
 - var useBadges = docType || stability;
-
-// renamer :: String -> String
-// Renames `Let` and `Var` into `Const`
+//- renamer :: String -> String
+//- Renames `Let` and `Var` into `Const`
 - var renamer = function renamer(docType) {
 -   return (docType === 'Let' || docType === 'Var') ? 'Const' : docType
 - }
 
 if current.path[4] && current.path[3] == 'api'
   - var textFormat = 'is-standard-case'
 
-header(class="hero background-sky")
+header(class="hero background-sky", style=fixHeroCss ? "height:auto" : "")
   div(class="inner-header")
     h1(class="hero-title text-display-1 #{textFormat}") #{headerTitle}
     if useBadges
@@ -33,5 +33,7 @@ header(class="hero background-sky")
   if subtitle
     h2.hero-subtitle.text-subhead #{subtitle}
 
+  else if current.path[3] == 'api' && current.path[1] == 'dart'
+    block breadcrumbs
   else if current.path[0] == "docs"
     != partial("_version-dropdown")

public/docs/_layout-dart-api.jade

@@ -0,0 +1,40 @@
+//- WARNING: _layout.jade and _layout-dart-api.jade should match in terms of content
+//- except that one uses Harp partial/yield and the other uses Jade extends/include.
+doctype
+html(lang="en" ng-app="angularIOApp" itemscope itemtype="http://schema.org/Framework")
+  // template: public/docs/_layout-dart-api
+  head
+    include ../_includes/_head-include
+    block head-extra
+
+  block var-def
+  body(class="l-offset-nav l-offset-side-nav"  ng-controller="AppCtrl as appCtrl")
+    include ../_includes/_main-nav
+    if current.path[2]
+      include _includes/_side-nav
+    include ../_includes/_hero
+    include ../_includes/_banner
+
+    if current.path[3] == 'api'
+      if current.path[4] == 'index'
+        block main-content
+      else
+        article(class="l-content-small grid-fluid docs-content")
+          block main-content
+    else if current.path.indexOf('cheatsheet') > 0
+      block main-content
+    else
+      if current.path[3] == 'index' || current.path[3] == 'styleguide'
+        article(class="l-content-small grid-fluid docs-content")
+          block main-content
+      else
+        article(class="l-content-small grid-fluid docs-content")
+          div(class="c10")
+            .showcase
+              .showcase-content
+                block main-content
+                if (current.path[3] == 'guide' || current.path[3] == 'tutorial') && current.path[4]
+                  include ../_includes/_next-item
+
+    include ../_includes/_footer
+    include ../_includes/_scripts-include

public/docs/_layout.jade

@@ -1,8 +1,13 @@
+//- WARNING: _layout.jade and _layout-dart-api.jade should match in terms of content
+//- except that one uses Harp partial/yield and the other uses Jade extends/include.
 doctype
 html(lang="en" ng-app="angularIOApp" itemscope itemtype="http://schema.org/Framework")
+  // template: public/docs/_layout
   head
     != partial("../_includes/_head-include")
+    block head-extra
 
+  //-
   body(class="l-offset-nav l-offset-side-nav"  ng-controller="AppCtrl as appCtrl")
     != partial("../_includes/_main-nav")
     if current.path[2]

public/docs/dart/latest/api/index.jade

@@ -1,10 +1,13 @@
-.l-main-section
-  h2 Beta
+:marked
+  > **WARNING:** API documentation is preliminary and subject to change.
 
-  p.
-    The proposed Angular 2 API does not yet have Dart-specific documentation.
-    However, because the Dart and JavaScript APIs are generated from the same source,
-    you might find the JavaScript API docs helpful:
+  > **Known issues:** Although this generated API reference displays Dart
+  APIs, individual pages sometimes describe TypeScript APIs accompanied with
+  TypeScript code. The angular.io issue tracker contains [all known
+  issues][api-issues]; if you notice others, please [report
+  them][new-issue]. Thanks!
 
-  p.text-center
-    <b><a href="/docs/js/latest/api/">Angular 2 API Preview (JavaScript)</a></b>
+  [new-issue]: https://github.com/angular/angular.io/issues/new?labels=dart,api&title=%5BDart%5D%5BAPI%5D%20
+  [api-issues]: https://github.com/angular/angular.io/issues?q=label%3Aapi+label%3Adart
+
+api-list(src="api-list.json" lang="dart")

public/resources/js/directives/api-list.js

@@ -26,6 +26,8 @@ angularIO.directive('apiList', function () {
     controller: function($scope, $attrs, $http, $location) {
       var $ctrl = this;
 
+      var isForDart = $attrs.lang === 'dart';
+
       $ctrl.apiTypes = [
         { cssClass: 'stable', title: 'Stable', matches: ['stable']},
         { cssClass: 'directive', title: 'Directive', matches: ['directive'] },
@@ -37,6 +39,9 @@ angularIO.directive('apiList', function () {
         { cssClass: 'const', title: 'Const', matches: ['var', 'let', 'const'] }
       ];
 
+      if (isForDart) $ctrl.apiTypes = $ctrl.apiTypes.filter((t) => 
+        !t.cssClass.match(/^(stable|directive|decorator|interface|enum)$/));
+
       $ctrl.apiFilter = getApiFilterFromLocation();
       $ctrl.apiType = getApiTypeFromLocation();
       $ctrl.groupedSections = [];

scripts/patch.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+set -e -o pipefail
+
+TARGET=node_modules/terraform/lib/helpers/raw.js
+
+# Around line 282 change from/to:
+#   var namespace = sourcePath.split(".")[0].split("/")
+#   var namespace = sourcePath.split('.').slice(0, -1).join('.').split('/')
+
+if [ -e "$TARGET" ]; then
+    perl -i.bak -pe 's/^(\s+var namespace.*split\("."\))\[0\]/\1.slice(0, -1).join(".")/' "$TARGET"
+    echo "Patched '$TARGET'."
+else
+    echo "Nothing to patch. Can't find file '$TARGET'."
+    exit 1;
+fi