feat(api-reference): add typedoc plugins for api reference (#1694)

* docs: add typedoc plugins and customizations

* docs: support generating typedoc for all packages

* docs: consolidate client doc generator plugins

* docs: add core packages doc plugin

* feat(client-documentation-generator): refactor plugins

* chore(core-packages-documentation-generator): generate core packages doc

* fix: apply review feedbacks

Co-authored-by: Trivikram Kamat <[email protected]>

Author: AllanZhengYP

Diff for: package.json

@@ -11,10 +11,12 @@
     "clean": "yarn clear-build-cache && yarn clear-build-info && lerna clean",
     "clear-build-cache": "rimraf ./packages/*/build  ./packages/*/build-es ./clients/*/dist",
     "clear-build-info": "rimraf ./packages/*/*.tsbuildinfo ./clients/*/*/*.tsbuildinfo",
+    "remove-documentation": "rimraf ./docs",
     "build:crypto-dependencies": "lerna run --scope '@aws-sdk/{types,util-utf8-browser,util-locate-window,hash-node}' --include-dependencies pretest",
     "build:protocols": "yarn build:crypto-dependencies && lerna run --scope '@aws-sdk/aws-*' --include-dependencies pretest",
     "build:smithy-client": "yarn build:crypto-dependencies && lerna run --scope '@aws-sdk/client-rds-data' --include-dependencies pretest",
     "build:all": "yarn build:crypto-dependencies && lerna run build",
+    "build-documentation": "yarn remove-documentation && typedoc",
     "pretest:all": "yarn build:all",
     "test:all": "jest --coverage --passWithNoTests && lerna run test --scope '@aws-sdk/{fetch-http-handler,hash-blob-browser}'",
     "test:functional": "jest --config tests/functional/jest.config.js --passWithNoTests",
@@ -79,6 +81,7 @@
     "prettier": "2.1.0",
     "puppeteer": "^5.1.0",
     "ts-loader": "^7.0.5",
+    "typedoc-plugin-lerna-packages": "^0.3.1",
     "typescript": "~4.0.2",
     "verdaccio": "^4.7.2",
     "webpack": "^4.43.0",
@@ -112,4 +115,4 @@
     ],
     "**/*.{ts,js,md,json}": "prettier --write"
   }
-}
+}

Diff for: packages/client-documentation-generator/src/index.ts

@@ -1,20 +1,20 @@
 import { PluginHost } from "typedoc/dist/lib/utils";
 
-import { SdkClientRenameGlobalPlugin } from "./sdk-client-rename-global";
-import { SdkClientSourceUpdatePlugin } from "./sdk-client-source-update";
+import { SdkClientCommentUpdatePlugin } from "./sdk-client-comment-update";
+import { SdkClientRenameProjectPlugin } from "./sdk-client-rename-project";
 import { SdkClientTocPlugin } from "./sdk-client-toc-plugin";
 
-/**
- *
- * @param pluginHost An instance of PluginHost.
- */
 module.exports = function load(pluginHost: PluginHost) {
   const application = pluginHost.owner;
 
-  // Add renderer plugins
-  application.renderer.addComponent("SdkClientTocPlugin", SdkClientTocPlugin as any);
-  application.renderer.addComponent("SdkClientRenameGlobalPlugin", SdkClientRenameGlobalPlugin as any);
+  application.converter.addComponent(
+    "SdkClientCommentUpdatePlugin",
+    new SdkClientCommentUpdatePlugin(application.converter)
+  );
 
-  // Add converter plugins
-  application.converter.addComponent("SdkClientSourceUpdatePlugin", SdkClientSourceUpdatePlugin as any);
+  application.renderer.addComponent("SdkClientTocPlugin", new SdkClientTocPlugin(application.renderer));
+  application.renderer.addComponent(
+    "SdkClientRenameProjectPlugin",
+    new SdkClientRenameProjectPlugin(application.renderer)
+  );
 };

Diff for: packages/client-documentation-generator/src/sdk-client-comment-update.ts

@@ -0,0 +1,38 @@
+import { Converter } from "typedoc/dist/lib/converter";
+import { Component, ConverterComponent } from "typedoc/dist/lib/converter/components";
+import { Context } from "typedoc/dist/lib/converter/context";
+import { getRawComment, parseComment } from "typedoc/dist/lib/converter/factories/comment";
+import { Reflection } from "typedoc/dist/lib/models/reflections";
+import ts from "typescript";
+
+/**
+ * Best effort make the service docs markdown looks better.
+ */
+@Component({ name: "SdkClientCommentUpdatePlugin" })
+export class SdkClientCommentUpdatePlugin extends ConverterComponent {
+  initialize() {
+    this.listenTo(this.owner, {
+      [Converter.EVENT_CREATE_DECLARATION]: this.onDeclaration,
+    });
+  }
+
+  private onDeclaration(context: Context, reflection: Reflection, node?: ts.Node) {
+    if (!node) return;
+    const rawComment = getRawComment(node);
+    if (!rawComment) return;
+    const comment = parseComment(this.cleanEmptyCommentLines(rawComment));
+    reflection.comment = comment;
+  }
+
+  /**
+   * Update documentation block to exclude empty lines.
+   */
+  private cleanEmptyCommentLines(comment: string): string {
+    return comment.startsWith("/*") && comment.endsWith("*/")
+      ? comment
+          .split("\n")
+          .filter((line) => line.substr(line.indexOf("*") + 1).trim().length !== 0)
+          .join("\n")
+      : comment;
+  }
+}

Diff for: packages/client-documentation-generator/src/sdk-client-rename-global.ts

@@ -0,0 +1,23 @@
+import { readFileSync } from "fs";
+import { Component, RendererComponent } from "typedoc/dist/lib/output/components";
+import { RendererEvent } from "typedoc/dist/lib/output/events";
+
+/**
+ * Correct the package name in the navigator.
+ */
+@Component({ name: "SdkClientRenameProject" })
+export class SdkClientRenameProjectPlugin extends RendererComponent {
+  initialize() {
+    this.listenTo(this.owner, {
+      [RendererEvent.BEGIN]: this.onRenderedBegin,
+    });
+  }
+
+  onRenderedBegin(event: RendererEvent) {
+    const { fullFileName } = event.project.files.filter((sourceFile) =>
+      sourceFile.fileName.endsWith("/package.json")
+    )[0];
+    const { name } = JSON.parse(readFileSync(fullFileName).toString());
+    event.project.name = name;
+  }
+}

Diff for: packages/client-documentation-generator/src/sdk-client-rename-project.ts

@@ -4,11 +4,14 @@ import { Component, RendererComponent } from "typedoc/dist/lib/output/components
 import { PageEvent } from "typedoc/dist/lib/output/events";
 import { NavigationItem } from "typedoc/dist/lib/output/models/NavigationItem";
 
+/**
+ * Group the ToC for easier observability.
+ */
 @Component({ name: "SdkClientTocPlugin" })
 export class SdkClientTocPlugin extends RendererComponent {
-  private commandToNavigationItems: Map<string, NavigationItem> = new Map();
   private commandsNavigationItem?: NavigationItem;
-  private exceptionsNavigationItem?: NavigationItem;
+  private clientsNavigationItem?: NavigationItem;
+  private paginatorsNavigationItem?: NavigationItem;
 
   initialize() {
     // disable existing toc plugin
@@ -40,36 +43,43 @@ export class SdkClientTocPlugin extends RendererComponent {
     page.toc = new NavigationItem(model.name);
 
     if (!model.parent && !trail.length) {
+      this.clientsNavigationItem = new NavigationItem("Clients", void 0, page.toc);
       this.commandsNavigationItem = new NavigationItem("Commands", void 0, page.toc);
-      this.exceptionsNavigationItem = new NavigationItem("Exceptions", void 0, page.toc);
+      this.paginatorsNavigationItem = new NavigationItem("Paginators", void 0, page.toc);
     }
 
     this.buildToc(model, trail, page.toc, tocRestriction);
   }
 
-  private isCommand({ implementedTypes = [] }: DeclarationReflection): boolean {
+  private isClient(model: DeclarationReflection): boolean {
+    const { extendedTypes = [] } = model;
     return (
-      implementedTypes.length === 1 &&
-      implementedTypes[0].type === "reference" &&
-      (implementedTypes[0] as ReferenceType).name === "Command"
+      model.kindOf(ReflectionKind.Class) &&
+      model.getFullName() !== "Client" && // Exclude the Smithy Client class.
+      (model.name.endsWith("Client") /* Modular client like S3Client */ ||
+        (extendedTypes.length === 1 &&
+          (extendedTypes[0] as ReferenceType).name.endsWith("Client"))) /* Legacy client like S3 */
     );
   }
 
-  private isException(model: DeclarationReflection): boolean {
-    const extendedTypes = model.extendedTypes || [];
+  private isCommand(model: DeclarationReflection): boolean {
     return (
-      extendedTypes.length === 1 &&
-      extendedTypes[0].type === "reference" &&
-      (extendedTypes[0] as ReferenceType).name === "ServiceException"
+      model.kindOf(ReflectionKind.Class) &&
+      model.getFullName() !== "Command" && // Exclude the Smithy Command class.
+      model.name.endsWith("Command") &&
+      model.children?.some((child) => child.name === "resolveMiddleware")
     );
   }
 
-  private isUnion(model: DeclarationReflection): boolean {
-    return model.type?.type === "union";
+  private isPaginator(model: DeclarationReflection): boolean {
+    return model.name.startsWith("paginate") && model.kindOf(ReflectionKind.Function);
   }
 
   private isInputOrOutput(model: DeclarationReflection): boolean {
-    return model.kindString === "Interface" && (model.name.endsWith("Input") || model.name.endsWith("Output"));
+    return (
+      model.kindOf(ReflectionKind.TypeAlias) &&
+      (model.name.endsWith("CommandInput") || model.name.endsWith("CommandOutput"))
+    );
   }
 
   /**
@@ -92,57 +102,33 @@ export class SdkClientTocPlugin extends RendererComponent {
       this.buildToc(child, trail, item);
     } else {
       children.forEach((child: DeclarationReflection) => {
-        if (restriction && restriction.length > 0 && restriction.indexOf(child.name) === -1) {
+        if (restriction && restriction.length > 0 && !restriction.includes(child.name)) {
           return;
         }
 
         if (child.kindOf(ReflectionKind.SomeModule)) {
           return;
         }
 
-        if (trail.length) {
-          const item = NavigationItem.create(child, parent, true);
-          if (trail.indexOf(child) !== -1) {
-            item.isInPath = true;
-            item.isCurrent = trail[trail.length - 1] === child;
-            this.buildToc(child, trail, item);
-          }
-          return;
-        }
-
-        if (this.isCommand(child)) {
-          const item = NavigationItem.create(child, this.commandsNavigationItem, true);
-          // create an entry for the command
-          const commandName = child.name.toLowerCase();
-          if (!this.commandToNavigationItems.get(commandName)) {
-            this.commandToNavigationItems.set(commandName, item);
-          }
-        } else if (this.isException(child)) {
-          NavigationItem.create(child, this.exceptionsNavigationItem, true);
-        } else if (
-          this.isUnion(child) &&
-          (child as any).type.types.every((type: ReferenceType) => {
-            return type.reflection && this.isException(type.reflection as DeclarationReflection);
-          })
-        ) {
-          // get command from name
-          const commandName = child.name.replace("ExceptionsUnion", "").toLowerCase() + "command";
-          NavigationItem.create(child, this.commandToNavigationItems.get(commandName), true);
+        if (this.isClient(child)) {
+          NavigationItem.create(child, this.clientsNavigationItem, true);
+        } else if (this.isCommand(child)) {
+          NavigationItem.create(child, this.commandsNavigationItem, true);
+        } else if (this.isPaginator(child)) {
+          NavigationItem.create(child, this.paginatorsNavigationItem, true);
         } else if (this.isInputOrOutput(child)) {
-          // get command from name
-          const commandName = child.name.replace(/Input|Output/, "").toLowerCase() + "command";
-          NavigationItem.create(child, this.commandToNavigationItems.get(commandName), true);
-        } else if (child.name.startsWith("_")) {
-          return;
+          NavigationItem.create(child, this.commandsNavigationItem, true);
         } else {
           const item = NavigationItem.create(child, parent, true);
-          if (trail.indexOf(child) !== -1) {
+          if (trail.includes(child)) {
             item.isInPath = true;
             item.isCurrent = trail[trail.length - 1] === child;
             this.buildToc(child, trail, item);
           }
         }
       });
+      // Group commands and input/output interface of each command.
+      this.commandsNavigationItem?.children.sort((childA, childB) => childA.title.localeCompare(childB.title));
     }
   }
 }