fix(typescript-estree): handle running out of fs watchers (#1088)

Author: bradzacher

Diff for: packages/typescript-estree/README.md

@@ -144,6 +144,11 @@ I work closely with the TypeScript Team and we are gradually aliging the AST of
 - `npm run unit-tests` - run only unit tests
 - `npm run ast-alignment-tests` - run only Babylon AST alignment tests
 
+## Debugging
+
+If you encounter a bug with the parser that you want to investigate, you can turn on the debug logging via setting the environment variable: `DEBUG=typescript-eslint:*`.
+I.e. in this repo you can run: `DEBUG=typescript-eslint:* yarn lint`.
+
 ## License
 
 TypeScript ESTree inherits from the the original TypeScript ESLint Parser license, as the majority of the work began there. It is licensed under a permissive BSD 2-clause license.

Diff for: packages/typescript-estree/package.json

@@ -40,6 +40,7 @@
   },
   "dependencies": {
     "chokidar": "^3.0.2",
+    "debug": "^4.1.1",
     "glob": "^7.1.4",
     "is-glob": "^4.0.1",
     "lodash.unescape": "4.0.1",
@@ -50,6 +51,7 @@
     "@babel/parser": "7.5.5",
     "@babel/types": "^7.3.2",
     "@types/babel-code-frame": "^6.20.1",
+    "@types/debug": "^4.1.5",
     "@types/glob": "^7.1.1",
     "@types/is-glob": "^4.0.1",
     "@types/lodash.isplainobject": "^4.0.4",

Diff for: packages/typescript-estree/src/parser.ts

@@ -1,3 +1,4 @@
+import debug from 'debug';
 import path from 'path';
 import semver from 'semver';
 import * as ts from 'typescript'; // leave this as * as ts so people using util package don't need syntheticDefaultImports
@@ -15,6 +16,8 @@ import {
   defaultCompilerOptions,
 } from './tsconfig-parser';
 
+const log = debug('typescript-eslint:typescript-estree:parser');
+
 /**
  * This needs to be kept in sync with the top-level README.md in the
  * typescript-eslint monorepo
@@ -41,6 +44,17 @@ function getFileName({ jsx }: { jsx?: boolean }): string {
   return jsx ? 'estree.tsx' : 'estree.ts';
 }
 
+function enforceString(code: unknown): string {
+  /**
+   * Ensure the source code is a string
+   */
+  if (typeof code !== 'string') {
+    return String(code);
+  }
+
+  return code;
+}
+
 /**
  * Resets the extra config object
  */
@@ -82,6 +96,8 @@ function getASTFromProject(
   options: TSESTreeOptions,
   createDefaultProgram: boolean,
 ): ASTAndProgram | undefined {
+  log('Attempting to get AST from project(s) for: %s', options.filePath);
+
   const filePath = options.filePath || getFileName(options);
   const astAndProgram = firstDefined(
     calculateProjectParserOptions(code, filePath, extra),
@@ -139,6 +155,11 @@ function getASTAndDefaultProject(
   code: string,
   options: TSESTreeOptions,
 ): ASTAndProgram | undefined {
+  log(
+    'Attempting to get AST from the default project(s): %s',
+    options.filePath,
+  );
+
   const fileName = options.filePath || getFileName(options);
   const program = createProgram(code, fileName, extra);
   const ast = program && program.getSourceFile(fileName);
@@ -150,6 +171,8 @@ function getASTAndDefaultProject(
  * @returns Returns a new source file and program corresponding to the linted code
  */
 function createNewProgram(code: string): ASTAndProgram {
+  log('Getting AST without type information');
+
   const FILENAME = getFileName(extra);
 
   const compilerHost: ts.CompilerHost = {
@@ -226,6 +249,9 @@ function getProgramAndAST(
 }
 
 function applyParserOptionsToExtra(options: TSESTreeOptions): void {
+  /**
+   * Turn on/off filesystem watchers
+   */
   extra.noWatch = typeof options.noWatch === 'boolean' && options.noWatch;
 
   /**
@@ -378,6 +404,7 @@ export function parse<T extends TSESTreeOptions = TSESTreeOptions>(
    * Reset the parse configuration
    */
   resetExtra();
+
   /**
    * Ensure users do not attempt to use parse() when they need parseAndGenerateServices()
    */
@@ -386,24 +413,25 @@ export function parse<T extends TSESTreeOptions = TSESTreeOptions>(
       `"errorOnTypeScriptSyntacticAndSemanticIssues" is only supported for parseAndGenerateServices()`,
     );
   }
+
   /**
    * Ensure the source code is a string, and store a reference to it
    */
-  // eslint-disable-next-line @typescript-eslint/no-explicit-any
-  if (typeof code !== 'string' && !((code as any) instanceof String)) {
-    code = String(code);
-  }
+  code = enforceString(code);
   extra.code = code;
+
   /**
    * Apply the given parser options
    */
   if (typeof options !== 'undefined') {
     applyParserOptionsToExtra(options);
   }
+
   /**
    * Warn if the user is using an unsupported version of TypeScript
    */
   warnAboutTSVersion();
+
   /**
    * Create a ts.SourceFile directly, no ts.Program is needed for a simple
    * parse
@@ -414,6 +442,7 @@ export function parse<T extends TSESTreeOptions = TSESTreeOptions>(
     ts.ScriptTarget.Latest,
     /* setParentNodes */ true,
   );
+
   /**
    * Convert the TypeScript AST to an ESTree-compatible one
    */
@@ -428,14 +457,13 @@ export function parseAndGenerateServices<
    * Reset the parse configuration
    */
   resetExtra();
+
   /**
    * Ensure the source code is a string, and store a reference to it
    */
-  // eslint-disable-next-line @typescript-eslint/no-explicit-any
-  if (typeof code !== 'string' && !((code as any) instanceof String)) {
-    code = String(code);
-  }
+  code = enforceString(code);
   extra.code = code;
+
   /**
    * Apply the given parser options
    */
@@ -449,10 +477,12 @@ export function parseAndGenerateServices<
       extra.errorOnTypeScriptSyntacticAndSemanticIssues = true;
     }
   }
+
   /**
    * Warn if the user is using an unsupported version of TypeScript
    */
   warnAboutTSVersion();
+
   /**
    * Generate a full ts.Program in order to be able to provide parser
    * services, such as type-checking
@@ -465,6 +495,7 @@ export function parseAndGenerateServices<
     shouldProvideParserServices,
     extra.createDefaultProgram,
   )!;
+
   /**
    * Determine whether or not two-way maps of converted AST nodes should be preserved
    * during the conversion process
@@ -473,11 +504,13 @@ export function parseAndGenerateServices<
     extra.preserveNodeMaps !== undefined
       ? extra.preserveNodeMaps
       : shouldProvideParserServices;
+
   /**
    * Convert the TypeScript AST to an ESTree-compatible one, and optionally preserve
    * mappings between converted and original AST nodes
    */
   const { estree, astMaps } = astConverter(ast, extra, shouldPreserveNodeMaps);
+
   /**
    * Even if TypeScript parsed the source code ok, and we had no problems converting the AST,
    * there may be other syntactic or semantic issues in the code that we can optionally report on.
@@ -488,6 +521,7 @@ export function parseAndGenerateServices<
       throw convertError(error);
     }
   }
+
   /**
    * Return the converted AST and additional parser services
    */

Diff for: packages/typescript-estree/src/tsconfig-parser.ts

@@ -1,12 +1,11 @@
 import chokidar from 'chokidar';
+import debug from 'debug';
 import path from 'path';
 import * as ts from 'typescript'; // leave this as * as ts so people using util package don't need syntheticDefaultImports
 import { Extra } from './parser-options';
 import { WatchCompilerHostOfConfigFile } from './WatchCompilerHostOfConfigFile';
 
-//------------------------------------------------------------------------------
-// Environment calculation
-//------------------------------------------------------------------------------
+const log = debug('typescript-eslint:typescript-estree:tsconfig-parser');
 
 /**
  * Default compiler options for program generation from single root file
@@ -33,16 +32,18 @@ const knownWatchProgramMap = new Map<
  */
 const watchCallbackTrackingMap = new Map<string, Set<ts.FileWatcherCallback>>();
 
-/**
- * Tracks the ts.sys.watchFile watchers that we've opened for config files.
- * We store these so we can clean up our handles if required.
- */
-const configSystemFileWatcherTrackingSet = new Set<ts.FileWatcher>();
+interface Watcher {
+  close(): void;
+  forceClose(): void;
+  on(evt: 'add', listener: (file: string) => void): void;
+  on(evt: 'change', listener: (file: string) => void): void;
+  trackWatcher(): void;
+}
 /**
  * Tracks the ts.sys.watchDirectory watchers that we've opened for project folders.
  * We store these so we can clean up our handles if required.
  */
-const directorySystemFileWatcherTrackingSet = new Set<ts.FileWatcher>();
+const fileWatcherTrackingSet = new Map<string, Watcher>();
 
 const parsedFilesSeen = new Set<string>();
 
@@ -56,12 +57,8 @@ export function clearCaches(): void {
   parsedFilesSeen.clear();
 
   // stop tracking config files
-  configSystemFileWatcherTrackingSet.forEach(cb => cb.close());
-  configSystemFileWatcherTrackingSet.clear();
-
-  // stop tracking folders
-  directorySystemFileWatcherTrackingSet.forEach(cb => cb.close());
-  directorySystemFileWatcherTrackingSet.clear();
+  fileWatcherTrackingSet.forEach(cb => cb.forceClose());
+  fileWatcherTrackingSet.clear();
 }
 
 /**
@@ -88,34 +85,84 @@ function getTsconfigPath(tsconfigPath: string, extra: Extra): string {
     : path.join(extra.tsconfigRootDir || process.cwd(), tsconfigPath);
 }
 
-interface Watcher {
-  close(): void;
-  on(evt: 'add', listener: (file: string) => void): void;
-  on(evt: 'change', listener: (file: string) => void): void;
-}
+const EMPTY_WATCHER: Watcher = {
+  close: (): void => {},
+  forceClose: (): void => {},
+  on: (): void => {},
+  trackWatcher: (): void => {},
+};
+
 /**
  * Watches a file or directory for changes
  */
 function watch(
-  path: string,
+  watchPath: string,
   options: chokidar.WatchOptions,
   extra: Extra,
 ): Watcher {
   // an escape hatch to disable the file watchers as they can take a bit to initialise in some cases
   // this also supports an env variable so it's easy to switch on/off from the CLI
-  if (process.env.PARSER_NO_WATCH === 'true' || extra.noWatch === true) {
-    return {
-      close: (): void => {},
-      on: (): void => {},
-    };
+  const blockWatchers =
+    process.env.PARSER_NO_WATCH === 'false'
+      ? false
+      : process.env.PARSER_NO_WATCH === 'true' || extra.noWatch === true;
+  if (blockWatchers) {
+    return EMPTY_WATCHER;
+  }
+
+  // reuse watchers in case typescript asks us to watch the same file/directory multiple times
+  if (fileWatcherTrackingSet.has(watchPath)) {
+    const watcher = fileWatcherTrackingSet.get(watchPath)!;
+    watcher.trackWatcher();
+    return watcher;
   }
 
-  return chokidar.watch(path, {
-    ignoreInitial: true,
-    persistent: false,
-    useFsEvents: false,
-    ...options,
-  });
+  let fsWatcher: chokidar.FSWatcher;
+  try {
+    log('setting up watcher on path: %s', watchPath);
+    fsWatcher = chokidar.watch(watchPath, {
+      ignoreInitial: true,
+      persistent: false,
+      useFsEvents: false,
+      ...options,
+    });
+  } catch (e) {
+    log(
+      'error occurred using file watcher, setting up polling watcher instead: %s',
+      watchPath,
+    );
+    // https://github.com/microsoft/TypeScript/blob/c9d407b52ad92370cd116105c33d618195de8070/src/compiler/sys.ts#L1232-L1237
+    // Catch the exception and use polling instead
+    // Eg. on linux the number of watches are limited and one could easily exhaust watches and the exception ENOSPC is thrown when creating watcher at that point
+    // so instead of throwing error, use fs.watchFile
+    fsWatcher = chokidar.watch(watchPath, {
+      ignoreInitial: true,
+      persistent: false,
+      useFsEvents: false,
+      ...options,
+      usePolling: true,
+    });
+  }
+
+  let counter = 1;
+  const watcher = {
+    close: (): void => {
+      counter -= 1;
+      if (counter <= 0) {
+        fsWatcher.close();
+        fileWatcherTrackingSet.delete(watchPath);
+      }
+    },
+    forceClose: fsWatcher.close.bind(fsWatcher),
+    on: fsWatcher.on.bind(fsWatcher),
+    trackWatcher: (): void => {
+      counter += 1;
+    },
+  };
+
+  fileWatcherTrackingSet.set(watchPath, watcher);
+
+  return watcher;
 }
 
 /**
@@ -219,7 +266,6 @@ export function calculateProjectParserOptions(
         watcher.on('change', path => {
           callback(path, ts.FileWatcherEventKind.Changed);
         });
-        configSystemFileWatcherTrackingSet.add(watcher);
       }
 
       const normalizedFileName = path.normalize(fileName);
@@ -239,7 +285,6 @@ export function calculateProjectParserOptions(
 
           if (watcher) {
             watcher.close();
-            configSystemFileWatcherTrackingSet.delete(watcher);
           }
         },
       };
@@ -263,13 +308,9 @@ export function calculateProjectParserOptions(
       watcher.on('add', path => {
         callback(path);
       });
-      directorySystemFileWatcherTrackingSet.add(watcher);
 
       return {
-        close(): void {
-          watcher.close();
-          directorySystemFileWatcherTrackingSet.delete(watcher);
-        },
+        close: watcher.close,
       };
     };