feat(*): speed-up compilation of dynamic requires + total speedup

Author: s-panferov

src/deps.ts

@@ -87,6 +87,8 @@ function isIgnoreDependency(absulutePath: string) {
     return absulutePath == '%%ignore';
 }
 
+let lock: Promise<any>;
+
 export class FileAnalyzer {
     dependencies = new DependencyManager();
     validFiles = new ValidFilesManager();
@@ -96,9 +98,35 @@ export class FileAnalyzer {
         this.state = state;
     }
 
+    async checkDependenciesLocked(resolver: IResolver, fileName: string): Promise<boolean> {
+        let isValid = this.validFiles.isFileValid(fileName);
+        if (isValid) {
+            return isValid;
+        }
+
+        if (lock) {
+            return lock
+                .then(() => {
+                    return this.checkDependenciesLocked(resolver, fileName);
+                });
+        }
+
+        let resolveLock;
+        lock = new Promise((res, rej) => { resolveLock = res; });
+
+        try {
+            let checked = await this.checkDependencies(resolver, fileName);
+            return checked;
+        } finally {
+            lock = null;
+            resolveLock();
+        }
+    }
+
     async checkDependencies(resolver: IResolver, fileName: string): Promise<boolean> {
-        if (this.validFiles.isFileValid(fileName)) {
-            return false;
+        let isValid = this.validFiles.isFileValid(fileName);
+        if (isValid) {
+            return isValid;
         }
 
         this.validFiles.markFileValid(fileName);
@@ -110,7 +138,6 @@ export class FileAnalyzer {
             if (!this.state.hasFile(fileName)) {
                 changed = await this.state.readFileAndUpdate(fileName);
             }
-
             await this.checkDependenciesInternal(resolver, fileName);
         } catch (err) {
             this.validFiles.markFileInvalid(fileName);
@@ -350,8 +377,8 @@ export class DependencyManager {
 export class ValidFilesManager {
     files: {[fileName: string]: boolean} = {};
 
-    isFileValid(fileName: string): boolean {
-        return !!this.files[fileName];
+    isFileValid(fileName: string): Promise<boolean> | boolean {
+        return this.files[fileName];
     }
 
     markFileValid(fileName: string) {

src/host.ts

@@ -306,6 +306,26 @@ export class State {
         }
     }
 
+    fastEmit(fileName: string) {
+        fileName = this.normalizePath(fileName);
+
+        let file = this.getFile(fileName);
+        if (!file) {
+            throw new Error(`Unknown file ${ fileName }`);
+        }
+
+        let transpileResult = this.ts.transpileModule(file.text, {
+            compilerOptions: this.options,
+            reportDiagnostics: false,
+            fileName
+        });
+
+        return {
+            text: transpileResult.outputText,
+            sourceMap: transpileResult.sourceMapText
+        };
+    }
+
     updateFile(fileName: string, text: string, checked: boolean = false): boolean {
         fileName = this.normalizePath(fileName);
         let prevFile = this.files[fileName];
@@ -349,7 +369,8 @@ export class State {
     async readFile(fileName: string): Promise<string> {
         fileName = this.normalizePath(fileName);
         let buf = await this.readFileImpl(fileName);
-        return buf.toString('utf8');
+        let string = buf.toString('utf8');
+        return string;
     }
 
     readFileSync(fileName: string): string {

src/index.ts

@@ -84,9 +84,9 @@ async function compiler(webpack: IWebPack, text: string): Promise<void> {
     }
 
     try {
-        let wasChanged = await state.fileAnalyzer.checkDependencies(resolver, fileName);
+        let wasChanged = await state.fileAnalyzer.checkDependenciesLocked(resolver, fileName);
         if (wasChanged || doUpdate) {
-            state.updateProgram();
+            instance.shouldUpdateProgram = true;
         }
 
         let compiledModule;
@@ -109,26 +109,37 @@ async function compiler(webpack: IWebPack, text: string): Promise<void> {
             function transform() {
                 let resultText;
                 let resultSourceMap = null;
-                let output = state.emit(fileName);
 
-                let result = helpers.findResultFor(output, fileName);
+                if (state.options.declaration) {
+                    // can't use fastEmit with declaration generation
 
-                if (result.text === undefined) {
-                    throw new Error('No output found for ' + fileName);
-                }
+                    let output = state.emit(fileName);
+                    let result = helpers.findResultFor(output, fileName);
 
-                if (result.declaration) {
-                    webpack.emitFile(
-                        path.relative(process.cwd(), result.declaration.sourceName),
-                        result.declaration.text
-                    );
-                }
+                    if (result.text === undefined) {
+                        throw new Error('No output found for ' + fileName);
+                    }
+
+                    if (result.declaration) {
+                        webpack.emitFile(
+                            path.relative(process.cwd(), result.declaration.sourceName),
+                            result.declaration.text
+                        );
+                    }
 
-                resultText = result.text;
+                    resultText = result.text;
+                    resultSourceMap = result.sourceMap;
+                } else {
+                    // Use super-fast emit
+
+                    let result = state.fastEmit(fileName);
+                    resultText = result.text;
+                    resultSourceMap = result.sourceMap;
+                }
 
                 let sourceFileName = fileName.replace(process.cwd() + '/', '');
-                if (result.sourceMap) {
-                    resultSourceMap = JSON.parse(result.sourceMap);
+                if (resultSourceMap) {
+                    resultSourceMap = JSON.parse(resultSourceMap);
                     resultSourceMap.sources = [ sourceFileName ];
                     resultSourceMap.file = sourceFileName;
                     resultSourceMap.sourcesContent = [ text ];

src/instance.ts

@@ -35,6 +35,7 @@ export interface ICompilerInstance {
     plugins: LoaderPluginDef[];
     initedPlugins: LoaderPlugin[];
     externalsInvocation: Promise<any>;
+    shouldUpdateProgram: boolean;
 }
 
 interface ICompiler {
@@ -109,6 +110,7 @@ export function ensureInstance(webpack: IWebPack, options: ICompilerOptions, ins
     ensureInstanceStore(webpack._compiler);
 
     let exInstance = resolveInstance(webpack._compiler, instanceName);
+
     if (exInstance) {
         return exInstance;
     }
@@ -269,7 +271,8 @@ export function ensureInstance(webpack: IWebPack, options: ICompilerOptions, ins
         cacheIdentifier,
         plugins,
         initedPlugins,
-        externalsInvocation: null
+        externalsInvocation: null,
+        shouldUpdateProgram: true
     };
 }
 
@@ -298,13 +301,15 @@ function setupWatchRun(compiler, instanceName: string) {
                 if (EXTENSIONS.test(changedFile)) {
                     if (state.hasFile(changedFile)) {
                         await state.readFileAndUpdate(changedFile);
-                        await state.fileAnalyzer.checkDependencies(resolver, changedFile);
+                        await state.fileAnalyzer.checkDependenciesLocked(resolver, changedFile);
                     }
                 }
             });
 
             await Promise.all(tasks);
-            state.updateProgram();
+            if (!state.options.forkChecker) {
+                state.updateProgram();
+            }
             callback();
         } catch (err) {
             console.error(err.toString());
@@ -341,10 +346,11 @@ function setupAfterCompile(compiler, instanceName, forkChecker = false) {
 
             runChecker(instance, payload);
         } else {
-            if (!state.program) {
+            if (!state.program || instance.shouldUpdateProgram) {
                 // program may be undefined here, if all files
                 // will be loaded by tsconfig
                 state.updateProgram();
+                instance.shouldUpdateProgram = false;
             }
 
             let diagnostics = state.ts.getPreEmitDiagnostics(state.program);