feat: docgen refactoring (#545)

* feat(babel-plugin-function-filemeta): add first version of plugin

* feat(react-docgen-actual-name-handler): add new package

* chore(babel-plugin-export-metadata): change package

* feat(docz-core): add react docgen on data server

* chore: a lot of improvements

Author: pedronauck

core/docz-core/.babelrc

@@ -0,0 +1,3 @@
+{
+  "plugins": ["lodash"]
+}

core/docz-core/package.json

@@ -4,7 +4,6 @@
   "description": "All docz core logic of bundle and parsing is included on this package",
   "license": "MIT",
   "main": "dist/index.js",
-  "umd:main": "dist/index.umd.js",
   "module": "dist/index.m.js",
   "typings": "dist/index.d.ts",
   "source": "src/index.ts",
@@ -33,7 +32,7 @@
     "art-template": "^4.13.2",
     "babel-loader": "^8.0.2",
     "babel-preset-docz": "^0.13.5",
-    "babel-plugin-export-metadata": "^0.13.5",
+    "babel-plugin-export-metadata": "^0.13.4",
     "babylon": "^6.18.0",
     "cache-loader": "^1.2.5",
     "chalk": "^2.4.1",
@@ -64,7 +63,11 @@
     "poi": "^12.2.4",
     "progress-estimator": "^0.2.2",
     "react-dev-utils": "^6.1.1",
-    "react-docgen-typescript-loader": "^3.0.0-rc.0",
+    "react-docgen": "^2.21.0",
+    "react-docgen-actual-name-handler": "0.13.5",
+    "react-docgen-external-proptypes-handler": "^1.0.2",
+    "react-docgen-imported-proptype-handler": "^1.0.4",
+    "react-docgen-typescript": "^1.12.2",
     "react-hot-loader": "^4.6.3",
     "rehype-docz": "^0.13.5",
     "rehype-slug": "^2.0.2",
@@ -86,5 +89,6 @@
     "webpackbar": "^3.1.4",
     "ws": "^6.1.2",
     "yargs": "^12.0.5"
-  }
+  },
+  "devDependencies": {}
 }

core/docz-core/src/DataServer.ts

@@ -4,6 +4,7 @@ import { isFunction } from 'lodash/fp'
 import { touch } from './utils/fs'
 import * as paths from './config/paths'
 import { onSignal } from './utils/on-signal'
+import { promiseLogger } from './utils/promise-logger'
 
 export type Send = (type: string, payload: any) => void
 export type On = (type: string) => Promise<any>
@@ -26,6 +27,7 @@ export interface Params {
 }
 
 export interface State {
+  id: string
   init: (params: Params) => Promise<any>
   update: (params: Params) => any
   close: (params: Params) => any
@@ -56,14 +58,16 @@ export class DataServer {
 
   public async init(): Promise<void> {
     await Promise.all(
-      Array.from(this.states).map(
-        async state =>
-          isFunction(state.init) &&
-          state.init({
-            state: { ...this.state },
-            setState: this.setState(),
-          })
-      )
+      Array.from(this.states).map(async state => {
+        if (!isFunction(state.init)) return
+
+        const promise = state.init({
+          state: { ...this.state },
+          setState: this.setState(),
+        })
+
+        return promiseLogger(promise, `Initial data for ${state.id}`)
+      })
     )
 
     this.updateStateFile()

core/docz-core/src/commands/args.ts

@@ -24,6 +24,12 @@ const getInitialDescription = (pkg: any): string =>
 export type Env = 'production' | 'development'
 export type ThemeConfig = Record<string, any>
 
+export interface DocgenConfig {
+  handlers?: any[]
+  resolver?: (ast: any, recast: any) => any
+  propFilter?: (prop: any) => boolean
+}
+
 export interface Menu {
   name: string
   route?: string
@@ -89,6 +95,7 @@ export interface Config extends Argv {
   menu: Menu[]
   htmlContext: HtmlContext
   themeConfig: ThemeConfig
+  docgenConfig: DocgenConfig
   modifyBundlerConfig<C>(config: C, dev: boolean, args: Config): C
   modifyBabelRc(babelrc: BabelRC, args: Config): BabelRC
   onCreateWebpackChain<C>(c: C, dev: boolean, args: Config): void

core/docz-core/src/commands/build.ts

@@ -20,11 +20,12 @@ export const build = async (args: Config) => {
   const run = Plugin.runPluginsMethod(config.plugins)
   const dataServer = new DataServer()
 
+  if (args.propsParser) dataServer.register([states.props(config)])
   dataServer.register([states.config(config), states.entries(entries, config)])
 
   try {
     await promiseLogger(Entries.writeApp(config, true), 'Parsing mdx files')
-    await promiseLogger(dataServer.init(), 'Initializing data server')
+    await promiseLogger(dataServer.init(), 'Running data server')
 
     await promiseLogger(run('onPreBuild', config), 'Running onPreBuild()')
     await bundler.build(bundlerConfig)

core/docz-core/src/commands/dev.ts

@@ -42,13 +42,14 @@ export const dev = async (args: Config) => {
     config.websocketHost
   )
 
+  if (args.propsParser) dataServer.register([states.props(newConfig)])
   dataServer.register([
     states.config(newConfig),
     states.entries(entries, newConfig),
   ])
 
   try {
-    await promiseLogger(dataServer.init(), 'Initializing data server')
+    await promiseLogger(dataServer.init(), 'Running data server')
     await dataServer.listen()
   } catch (err) {
     logger.fatal('Failed to process your server:', err)

core/docz-core/src/config/babel.ts

@@ -32,22 +32,21 @@ export const getBabelConfig = async (
   const config = merge(localBabelRc, {
     presets,
     babelrc: false,
-    ...(args.debug && {
-      cacheDirectory: true,
-      cacheIdentifier: getCacheIdentifier(
-        isProd ? 'production' : isDev && 'development',
-        [
+    cacheCompression: args.debug ? false : isProd,
+    cacheDirectory: !args.debug,
+    cacheIdentifier: args.debug
+      ? null
+      : getCacheIdentifier(isProd ? 'production' : isDev && 'development', [
           'docz',
           'docz-theme-default',
           'docz-utils',
           'docz-core',
           'babel-preset-docz',
-        ]
-      ),
-    }),
-    cacheCompression: isProd,
+        ]),
     compact: isProd,
-    plugins: !isProd ? [require.resolve('react-hot-loader/babel')] : [],
+    plugins: [require.resolve('babel-plugin-export-metadata')].concat(
+      !isProd ? [require.resolve('react-hot-loader/babel')] : []
+    ),
   })
 
   const reduce = Plugin.reduceFromPlugins<BabelRC>(args.plugins)

core/docz-core/src/states/config.ts

@@ -3,10 +3,10 @@ import { load, finds } from 'load-cfg'
 import chokidar from 'chokidar'
 import get from 'lodash/get'
 
+import * as paths from '../config/paths'
 import { Params, State } from '../DataServer'
 import { Config, Menu, ThemeConfig } from '../commands/args'
 import { getRepoUrl } from '../utils/repo-info'
-import * as paths from '../config/paths'
 
 interface Payload {
   title: string
@@ -50,6 +50,7 @@ export const state = (config: Config): State => {
   watcher.setMaxListeners(Infinity)
 
   return {
+    id: 'config',
     init: updateConfig(config),
     close: () => watcher.close(),
     update: async params => {

core/docz-core/src/states/entries.ts

@@ -37,6 +37,7 @@ export const state = (entries: Entries, config: Config): State => {
   watcher.setMaxListeners(Infinity)
 
   return {
+    id: 'entries',
     init: updateEntries(entries),
     close: () => watcher.close(),
     update: async params => {

core/docz-core/src/states/index.ts

@@ -1,2 +1,3 @@
 export { state as entries } from './entries'
 export { state as config } from './config'
+export { state as props } from './props'

core/docz-core/src/states/props.ts

@@ -0,0 +1,64 @@
+import chokidar from 'chokidar'
+import equal from 'fast-deep-equal'
+import fastglob from 'fast-glob'
+import { State, Params } from '../DataServer'
+import { get, omit } from 'lodash/fp'
+
+import * as paths from '../config/paths'
+import { Config } from '../commands/args'
+import { docgen } from '../utils/docgen'
+
+const getPattern = (ts: boolean) => {
+  const tsPattern = '**/*.{ts,tsx}'
+  const jsPattern = '**/*.{js,jsx,mjs}'
+  return [ts ? tsPattern : jsPattern, '!**/node_modules']
+}
+
+const initial = (config: Config) => async (p: Params) => {
+  const pattern = getPattern(config.typescript)
+  const files = await fastglob<string>(pattern, { cwd: paths.root })
+  const metadata = await docgen(files, config)
+  p.setState('props', metadata)
+}
+
+const add = (config: Config) => (p: Params) => async (filepath: string) => {
+  const old = get('state.props', p)
+  const metadata = await docgen([filepath], config)
+
+  if (metadata && !equal(old, metadata)) {
+    p.setState('props', {
+      ...old,
+      ...metadata,
+    })
+  }
+}
+
+const remove = (config: Config) => (p: Params) => async (filepath: string) =>
+  p.setState('props', omit(filepath, get('state.props', p)))
+
+export const state = (config: Config): State => {
+  const pattern = getPattern(config.typescript)
+  const watcher = chokidar.watch(pattern, {
+    cwd: paths.root,
+    ignored: /(((^|[\/\\])\..+)|(node_modules))/,
+    persistent: true,
+  })
+
+  watcher.setMaxListeners(Infinity)
+
+  return {
+    id: 'props',
+    init: initial(config),
+    close: () => watcher.close(),
+    update: async params => {
+      const addFilepath = add(config)
+      const removeFilepath = remove(config)
+
+      watcher.on('add', addFilepath(params))
+      watcher.on('change', addFilepath(params))
+      watcher.on('unlink', removeFilepath(params))
+
+      return () => watcher.close()
+    },
+  }
+}

core/docz-core/src/types.d.ts

@@ -17,6 +17,9 @@ declare module 'humanize-string'
 declare module 'mini-html-webpack-plugin'
 declare module 'p-reduce'
 declare module 'progress-estimator'
+declare module 'react-docgen'
+declare module 'react-docgen-external-proptypes-handler'
+declare module 'react-docgen-imported-proptype-handler'
 declare module 'react-dev-utils/errorOverlayMiddleware'
 declare module 'react-dev-utils/evalSourceMapMiddleware'
 declare module 'react-dev-utils/FileSizeReporter'

core/docz-core/src/utils/docgen.ts

@@ -0,0 +1,68 @@
+import * as path from 'path'
+import * as fs from 'fs-extra'
+import { isFunction } from 'lodash/fp'
+import logger from 'signale'
+import findUp from 'find-up'
+import externalProptypesHandler from 'react-docgen-external-proptypes-handler'
+import importedProptypesHandler from 'react-docgen-imported-proptype-handler'
+import actualNameHandler from 'react-docgen-actual-name-handler'
+import reactDocgenTs from 'react-docgen-typescript'
+import reactDocgen from 'react-docgen'
+
+import * as paths from '../config/paths'
+import { Config } from '../commands/args'
+
+const fileFullPath = (filepath: string) => path.join(paths.root, filepath)
+
+const tsParser = async (files: string[], config: Config) => {
+  const tsConfigPath = await findUp('tsconfig.json', { cwd: paths.root })
+  if (!tsConfigPath) return {}
+
+  try {
+    const { parse } = reactDocgenTs.withCustomConfig(tsConfigPath, {
+      propFilter(prop: any, component: any): any {
+        if (prop.parent == null) return true
+        const propFilter = config.docgenConfig.propFilter
+        const val = propFilter && isFunction(propFilter) && propFilter(prop)
+        return !prop.parent.fileName.includes('node_modules') || Boolean(val)
+      },
+    })
+
+    return files
+      .map(filepath => ({ [fileFullPath(filepath)]: parse(filepath) }))
+      .reduce((obj, val) => ({ ...obj, ...val }), {})
+  } catch (err) {
+    logger.fatal('Error parsing static types.', err)
+    return {}
+  }
+}
+
+const jsParser = (files: string[], config: Config) => {
+  const resolver =
+    config.docgenConfig.resolver ||
+    reactDocgen.resolver.findAllExportedComponentDefinitions
+
+  return files.reduce((memo: any, filepath) => {
+    const handlers = reactDocgen.defaultHandlers.concat([
+      externalProptypesHandler(filepath),
+      importedProptypesHandler(filepath),
+      actualNameHandler,
+    ])
+
+    const code = fs.readFileSync(filepath, 'utf-8')
+
+    try {
+      const data = reactDocgen.parse(code, resolver, handlers)
+      memo[fileFullPath(filepath)] = data
+    } catch (err) {
+      if (err.message !== 'No suitable component definition found.') {
+        logger.fatal('Error:', filepath, err)
+      }
+    }
+
+    return memo
+  }, {})
+}
+
+export const docgen = async (files: string[], config: Config) =>
+  config.typescript ? tsParser(files, config) : jsParser(files, config)

core/docz-core/src/utils/load-config.ts

@@ -30,6 +30,7 @@ export const loadConfig = async (args: Config): Promise<Config> => {
       'license.md',
     ],
     themeConfig: {},
+    docgenConfig: {},
     modifyBundlerConfig: (config: any) => config,
     modifyBabelRc: (babelrc: BabelRC) => babelrc,
     onCreateWebpackChain: () => null,

core/docz-core/src/webpack/loaders.ts

@@ -54,8 +54,8 @@ export const mdx = (config: Config, args: Args) => {
     .end()
     .exclude.add(excludeNodeModules)
     .end()
-    .use('happypack-mdx')
-    .loader('happypack/loader?id=mdx')
+    .use('happypack-jsx')
+    .loader('happypack/loader?id=jsx')
     .end()
     .use('mdx-loader')
     .loader(require.resolve('@mdx-js/loader'))
@@ -81,33 +81,11 @@ export const setupHappypack = (config: Config, args: Args, babelrc: any) => {
     },
   ]
 
-  const loaderWithDocgen = [
-    args.propsParser &&
-      args.typescript && {
-        loader: require.resolve('react-docgen-typescript-loader'),
-        options: {
-          propFilter: (prop: any) => {
-            if (prop.parent == null) return true
-            return !prop.parent.fileName.includes('node_modules')
-          },
-        },
-      },
-  ]
-
   config.plugin('happypack-jsx').use(HappyPack, [
     {
       id: 'jsx',
       verbose: args.debug,
       threadPool: happyThreadPool,
-      loaders: loaders.concat(loaderWithDocgen).filter(Boolean),
-    },
-  ])
-
-  config.plugin('happypack-mdx').use(HappyPack, [
-    {
-      id: 'mdx',
-      verbose: args.debug,
-      threadPool: happyThreadPool,
       loaders: loaders.filter(Boolean),
     },
   ])