feat(*): implement file cache

Author: Stanislav Panferov

src/cache.ts

@@ -0,0 +1,148 @@
+import * as fs from 'fs';
+import * as crypto from 'crypto';
+import * as os from 'os';
+import * as path from 'path';
+import * as zlib from 'zlib';
+
+export interface CompiledModule {
+    fileName: string,
+    text: string,
+    map?: string,
+    mapName?: string
+}
+
+export function findCompiledModule(fileName: string): CompiledModule {
+    let baseFileName = fileName.replace(/(\.ts|\.tsx)$/, '');
+    let compiledFileName = `${baseFileName}.js`
+
+    if (fs.existsSync(compiledFileName)) {
+        let mapFileName = `${baseFileName}.js.map`;
+        let isMapExists = fs.existsSync(mapFileName);
+        let result = {
+            fileName: compiledFileName,
+            text: fs.readFileSync(compiledFileName).toString(),
+            mapName: isMapExists
+                ? mapFileName
+                : null,
+            map: isMapExists
+                ? fs.readFileSync(mapFileName).toString()
+                : null
+        }
+        return result;
+    } else {
+        return null;
+    }
+}
+
+/**
+ * Read the contents from the compressed file.
+ *
+ * @async
+ */
+function read(filename: string, callback) {
+    return fs.readFile(filename, function(err, data) {
+        if (err) { return callback(err); }
+
+        return zlib.gunzip(data, function(err, content) {
+            let result = {};
+
+            if (err) { return callback(err); }
+
+            try {
+                result = JSON.parse(content);
+            } catch (e) {
+                return callback(e);
+            }
+
+            return callback(null, result);
+        });
+    });
+};
+
+
+/**
+ * Write contents into a compressed file.
+ *
+ * @async
+ * @params {String} filename
+ * @params {String} result
+ * @params {Function} callback
+ */
+function write(filename: string, result: any, callback) {
+    let content = JSON.stringify(result);
+
+    return zlib.gzip(content as any, function(err, data) {
+        if (err) { return callback(err); }
+
+        return fs.writeFile(filename, data, callback);
+    });
+};
+
+
+/**
+ * Build the filename for the cached file
+ *
+ * @params {String} source  File source code
+ * @params {Object} options Options used
+ *
+ * @return {String}
+ */
+function filename(source: string, identifier, options) {
+    let hash = crypto.createHash('SHA1') as any;
+    let contents = JSON.stringify({
+        source: source,
+        options: options,
+        identifier: identifier,
+    });
+
+    hash.end(contents);
+
+    return hash.read().toString('hex') + '.json.gzip';
+};
+
+interface CacheParams {
+    source: string;
+    options: any;
+    transform: (source: string, options: any) => string;
+    identifier: any;
+    directory: string;
+}
+
+/**
+ * Retrieve file from cache, or create a new one for future reads
+ *
+ * @async
+ * @example
+ */
+export function cache(params: CacheParams, callback) {
+    // Spread params into named variables
+    // Forgive user whenever possible
+    let source = params.source;
+    let options = params.options || {};
+    let transform = params.transform;
+    let identifier = params.identifier;
+    let directory = (typeof params.directory === 'string') ?
+        params.directory :
+        os.tmpdir();
+    let file = path.join(directory, filename(source, identifier, options));
+
+    return read(file, function(err, content) {
+        let result = {};
+        // No errors mean that the file was previously cached
+        // we just need to return it
+        if (!err) { return callback(null, content); }
+
+        // Otherwise just transform the file
+        // return it to the user asap and write it in cache
+        try {
+            result = transform(source, options);
+        } catch (error) {
+            return callback(error);
+        }
+
+        return write(file, result, function(err) {
+            return callback(err, result);
+        });
+
+    });
+};

src/deps.ts

@@ -383,36 +383,6 @@ export class ValidFilesManager {
     }
 }
 
-export interface CompiledModule {
-    fileName: string,
-    text: string,
-    map?: string,
-    mapName?: string
-}
-
-export function findCompiledModule(fileName: string): CompiledModule {
-    let baseFileName = fileName.replace(/(\.ts|\.tsx)$/, '');
-    let compiledFileName = `${baseFileName}.js`
-
-    if (fs.existsSync(compiledFileName)) {
-        let mapFileName = `${baseFileName}.js.map`;
-        let isMapExists = fs.existsSync(mapFileName);
-        let result = {
-            fileName: compiledFileName,
-            text: fs.readFileSync(compiledFileName).toString(),
-            mapName: isMapExists
-                ? mapFileName
-                : null,
-            map: isMapExists
-                ? fs.readFileSync(mapFileName).toString()
-                : null
-        }
-        return result;
-    } else {
-        return null;
-    }
-}
-
 /**
  * Emit compilation result for a specified fileName.
  */

src/host.ts

@@ -39,6 +39,9 @@ export interface ICompilerOptions extends ts.CompilerOptions {
     doTypeCheck?: boolean;
     forkChecker?: boolean;
     useBabel?: boolean;
+    usePrecompiledFiles?: boolean;
+    useCache?: boolean;
+    cacheDirectory?: string;
 }
 
 export interface IOutputFile extends ts.OutputFile {

src/index.ts

@@ -9,12 +9,16 @@ import * as childProcess from 'child_process';
 import * as colors from 'colors';
 
 import { ICompilerOptions, TypeScriptCompilationError, State, ICompilerInfo } from './host';
-import { IResolver, ResolutionError, findCompiledModule } from './deps';
+import { IResolver, ResolutionError } from './deps';
+import { findCompiledModule, cache } from './cache';
 import * as helpers from './helpers';
 import { loadLib } from './helpers';
 
 let loaderUtils = require('loader-utils');
 
+let pkg = require('../package.json');
+let cachePromise = Promise.promisify(cache);
+
 interface ICompiler {
     inputFileSystem: typeof fs;
     _tsInstances: {[key:string]: ICompilerInstance};
@@ -44,6 +48,7 @@ interface ICompilerInstance {
     options: ICompilerOptions;
     externalsInvoked: boolean;
     checker: any;
+    cacheIdentifier: any;
 }
 
 function getRootCompiler(compiler) {
@@ -238,6 +243,28 @@ function ensureInstance(webpack: IWebPack, options: ICompilerOptions, instanceNa
         }
     }
 
+    let cacheIdentifier = null;
+    if (options.useCache) {
+        console.log(webpack.query);
+
+        if (!options.cacheDirectory) {
+            options.cacheDirectory = path.join(process.cwd(), '.awcache');
+        }
+
+        if (!fs.existsSync(options.cacheDirectory)) {
+            fs.mkdirSync(options.cacheDirectory)
+        }
+
+        cacheIdentifier = {
+            'typescript': tsImpl.version,
+            'awesome-typescript-loader': pkg.version,
+            'awesome-typescript-loader-query': webpack.query,
+            'babel-core': babelImpl
+                ? babelImpl.version
+                : null
+        }
+    }
+
     let tsState = new State(options, webpack._compiler.inputFileSystem, compilerInfo);
     let compiler = (<any>webpack._compiler);
 
@@ -322,7 +349,8 @@ function ensureInstance(webpack: IWebPack, options: ICompilerOptions, instanceNa
         externalsInvoked: false,
         checker: options.forkChecker
             ? createChecker(compilerInfo, options)
-            : null
+            : null,
+        cacheIdentifier
     }
 }
 
@@ -388,40 +416,69 @@ function compiler(webpack: IWebPack, text: string): void {
             });
         })
         .then(() => {
-            let resultText;
-            let resultSourceMap;
+            let compiledModule
+            if (instance.options.usePrecompiledFiles) {
+                compiledModule = findCompiledModule(fileName);
+            }
 
-            let compiledModule = findCompiledModule(fileName);
             if (compiledModule) {
                 state.fileAnalyzer.dependencies.addCompiledModule(fileName, compiledModule.fileName);
-                resultText = compiledModule.text;
-                resultSourceMap = JSON.parse(compiledModule.map);
+                return {
+                    text: compiledModule.text,
+                    map: JSON.parse(compiledModule.map)
+                }
             } else {
-                let output = state.emit(fileName);
-                let result = helpers.findResultFor(output, fileName);
 
-                if (result.text === undefined) {
-                    throw new Error('No output found for ' + fileName);
-                }
+                function transform() {
+                    let resultText;
+                    let resultSourceMap;
+                    let output = state.emit(fileName);
+                    let result = helpers.findResultFor(output, fileName);
 
-                resultText = result.text;
-                resultSourceMap = JSON.parse(result.sourceMap);
-                resultSourceMap.sources = [ fileName ];
-                resultSourceMap.file = fileName;
-                resultSourceMap.sourcesContent = [ text ];
+                    if (result.text === undefined) {
+                        throw new Error('No output found for ' + fileName);
+                    }
 
-                if (instance.options.useBabel) {
-                    let defaultOptions = {
-                        inputSourceMap: resultSourceMap,
-                        filename: fileName,
-                        sourceMap: true
+                    resultText = result.text;
+                    resultSourceMap = JSON.parse(result.sourceMap);
+                    resultSourceMap.sources = [ fileName ];
+                    resultSourceMap.file = fileName;
+                    resultSourceMap.sourcesContent = [ text ];
+
+                    if (instance.options.useBabel) {
+                        let defaultOptions = {
+                            inputSourceMap: resultSourceMap,
+                            filename: fileName,
+                            sourceMap: true
+                        }
+
+                        let babelResult = instance.babelImpl.transform(resultText, defaultOptions);
+                        resultText = babelResult.code;
+                        resultSourceMap = babelResult.map;
                     }
 
-                    let babelResult = instance.babelImpl.transform(resultText, defaultOptions);
-                    resultText = babelResult.code;
-                    resultSourceMap = babelResult.map;
+                    return {
+                        text: resultText,
+                        map: resultSourceMap
+                    };
+                }
+
+                if (instance.options.useCache) {
+                    return cachePromise({
+                        source: text,
+                        identifier: instance.cacheIdentifier,
+                        directory: instance.options.cacheDirectory,
+                        options: webpack.query,
+                        transform: transform
+                    })
+                } else {
+                    return transform();
                 }
             }
+        })
+        .then((transform: { text: string; map: any }) => {
+            let resultText = transform.text;
+            let resultSourceMap = transform.map;
 
             if (resultSourceMap) {
                 resultSourceMap.sources = [ fileName ];

src/tsconfig.json

@@ -16,6 +16,7 @@
         "!./node_modules/**/*.ts"
     ],
     "files": [
+        "./cache.ts",
         "./checker.ts",
         "./deps.ts",
         "./helpers.ts",