Merge pull request #80 from opichals/add-rollup

Add Rollup.js tree-shaking and terser minification

Author: gfwilliams

.gitignore

@@ -2,4 +2,5 @@
 .settings
 *~
 node_modules
+package-lock.json
 tmp

.npmignore

@@ -2,6 +2,7 @@
 CONTRIBUTING.md
 index.html
 libs/esprima
+libs/rollup/espruino-rollup.browser.js
 libs/targz.js
 libs/utf8.js
 plugins/_examplePlugin.js

bin/espruino-cli.js

@@ -254,6 +254,10 @@ function setupConfig(Espruino, callback) {
    process.exit(1);
    //Espruino.Core.Config.getSection(sectionName);
  }
+ if (args.file) {
+   var env = Espruino.Core.Env.getData();
+   env.FILE = args.file;
+ }
  if (args.board) {
    log("Explicit board JSON supplied: "+JSON.stringify(args.board));
    Espruino.Config.ENV_ON_CONNECT = false;

core/modules.js

@@ -61,8 +61,24 @@
 
     // When code is sent to Espruino, search it for modules and add extra code required to load them
     Espruino.addProcessor("transformForEspruino", function(code, callback) {
+      if (Espruino.Config.ROLLUP) {
+        return loadModulesRollup(code, callback);
+      }
       loadModules(code, callback);
     });
+
+    // Append the 'getModule' processor as the last (plugins get initialized after Espruino.Core modules)
+    Espruino.Plugins.CoreModules = {
+      init: function() {
+        Espruino.addProcessor("getModule", function(data, callback) {
+          if (data.moduleCode!==undefined) { // already provided be previous getModule processor
+            return callback(data);
+          }
+
+          fetchGetModule(data, callback);
+        });
+      }
+    };
   }
 
   function isBuiltIn(module) {
@@ -105,6 +121,50 @@
     return modules;
   };
 
+  /** Download modules from MODULE_URL/.. */
+  function fetchGetModule(data, callback) {
+    var fullModuleName = data.moduleName;
+
+    // try and load the module the old way...
+    console.log("loadModule("+fullModuleName+")");
+
+    var urls = []; // Array of where to look for this module
+    var modName; // Simple name of the module
+    if(Espruino.Core.Utils.isURL(fullModuleName)) {
+      modName = fullModuleName.substr(fullModuleName.lastIndexOf("/") + 1).split(".")[0];
+      urls = [ fullModuleName ];
+    } else {
+      modName = fullModuleName;
+      Espruino.Config.MODULE_URL.split("|").forEach(function (url) {
+        url = url.trim();
+        if (url.length!=0)
+        Espruino.Config.MODULE_EXTENSIONS.split("|").forEach(function (extension) {
+          urls.push(url + "/" + fullModuleName + extension);
+        })
+      });
+    };
+
+    // Recursively go through all the urls
+    (function download(urls) {
+      if (urls.length==0) {
+        return callback(data);
+      }
+      var dlUrl = urls[0];
+      Espruino.Core.Utils.getURL(dlUrl, function (code) {
+        if (code!==undefined) {
+          // we got it!
+          data.moduleCode = code;
+          data.isMinified = dlUrl.substr(-7)==".min.js";
+          return callback(data);
+        } else {
+          // else try next
+          download(urls.slice(1));
+        }
+      });
+    })(urls);
+  }
+
+
   /** Called from loadModule when a module is loaded. Parse it for other modules it might use
    *  and resolve dfd after all submodules have been loaded */
   function moduleLoaded(resolve, requires, modName, data, loadedModuleData, alreadyMinified){
@@ -144,51 +204,16 @@
     return new Promise(function(resolve, reject) {
       // First off, try and find this module using callProcessor
       Espruino.callProcessor("getModule",
-        { moduleName:fullModuleName, moduleCode:undefined },
+        { moduleName:fullModuleName, moduleCode:undefined, isMinified:false },
         function(data) {
-          if (data.moduleCode!==undefined) {
-            // great! it found something. Use it.
-            moduleLoaded(resolve, requires, fullModuleName, data.moduleCode, loadedModuleData, false);
-          } else {
-            // otherwise try and load the module the old way...
-            console.log("loadModule("+fullModuleName+")");
-
-            var urls = []; // Array of where to look for this module
-            var modName; // Simple name of the module
-            if(Espruino.Core.Utils.isURL(fullModuleName)) {
-              modName = fullModuleName.substr(fullModuleName.lastIndexOf("/") + 1).split(".")[0];
-              urls = [ fullModuleName ];
-            } else {
-              modName = fullModuleName;
-              Espruino.Config.MODULE_URL.split("|").forEach(function (url) {
-                url = url.trim();
-                if (url.length!=0)
-                  Espruino.Config.MODULE_EXTENSIONS.split("|").forEach(function (extension) {
-                    urls.push(url + "/" + fullModuleName + extension);
-                  })
-              });
-            };
-
-            // Recursively go through all the urls
-            (function download(urls) {
-              if (urls.length==0) {
-                Espruino.Core.Notifications.warning("Module "+fullModuleName+" not found");
-                return resolve();
-              }
-              var dlUrl = urls[0];
-              Espruino.Core.Utils.getURL(dlUrl, function (data) {
-                if (data!==undefined) {
-                  // we got it!
-                  moduleLoaded(resolve, requires, fullModuleName, data, loadedModuleData, dlUrl.substr(-7)==".min.js");
-                } else {
-                  // else try next
-                  download(urls.slice(1));
-                }
-              });
-            })(urls);
+          if (data.moduleCode===undefined) {
+            Espruino.Core.Notifications.warning("Module "+fullModuleName+" not found");
+            return resolve();
           }
-        });
 
+          // great! it found something. Use it.
+          moduleLoaded(resolve, requires, fullModuleName, data.moduleCode, loadedModuleData, data.isMinified);
+        });
     });
   }
 
@@ -211,8 +236,24 @@
         callback(loadedModuleData.join("\n") + "\n" + code);
       });
     }
-  };
+  }
+
+  function loadModulesRollup(code, callback) {
+    rollupTools.loadModulesRollup(code)
+      .then(generated => {
+        const minified = generated.code;
+        console.log('rollup: '+minified.length+' bytes');
 
+        // FIXME: needs warnings?
+        Espruino.Core.Notifications.info('Rollup no errors. Bundling ' + code.length + ' bytes to ' + minified.length + ' bytes');
+        callback(minified);
+      })
+      .catch(err => {
+        console.log('rollup:error', err);
+        Espruino.Core.Notifications.error("Rollup errors - Bundling failed: " + String(err).trim());
+        callback(code);
+      });
+  }
 
   Espruino.Core.Modules = {
     init : init

index.js

@@ -94,6 +94,13 @@ function init(callback) {
   // Various plugins
   loadDir(__dirname+"/plugins");
 
+  try {
+    global.espruinoRollup = require("./libs/rollup/espruino-rollup.js");
+    global.rollupTools = require("./libs/rollup/index.js");
+  } catch(e) {
+    console.log("espruinoRollup library not found - you'll need it to minify code");
+  }
+
   // Bodge up notifications  
   Espruino.Core.Notifications = {
     success : function(e) { console.log(e); },

libs/rollup/debug-shim.js

@@ -0,0 +1 @@
+export default () => () => undefined

libs/rollup/espruino-rollup.browser.js

@@ -0,0 +1,49 @@
+const { writeFileSync, vol } = require('fs');
+const rollup = require('rollup');
+const espruinoModules = require('rollup-plugin-espruino-modules');
+
+function bundle(options) {
+    const opts = { ...options };
+
+    if (typeof vol !== 'undefined') { // only in browser (fs = memfs)
+        vol.fromJSON({'/modules': null});
+
+        if (opts.modules) {
+            try {
+              opts.modules.forEach(([name, code]) => writeFileSync(name, code));
+            } catch (err) {
+              console.error('Write file failed:', err);
+            }
+            delete opts.modules;
+        }
+    }
+
+    const warnings = [];
+    opts.onwarn = warning => (warnings.push( warning ), (options.onwarn && options.onwarn(warning)));
+
+    const config = espruinoModules.buildRollupConfig(opts);
+
+    return rollup.rollup(config).then(bundle =>
+        bundle.generate(config.output).then(generated => {
+            generated.warnings = warnings;
+            return generated;
+        })
+    );
+}
+
+function minify(code, options) {
+    return new Promise((resolve, reject) => {
+        try {
+            const minifyOptions = espruinoModules.buildMinifyConfig(options)
+            const generated = espruinoModules.minify(code, minifyOptions);
+            resolve(generated);
+        } catch(e) {
+            reject(e);
+        }
+    });
+}
+
+module.exports = {
+    bundle,
+    minify
+}

libs/rollup/espruino-rollup.js

@@ -0,0 +1,98 @@
+(function (root, factory) {
+    'use strict';
+
+    if (typeof define === 'function' && define.amd) {
+        define(['exports'], factory);
+    } else if (typeof exports !== 'undefined') {
+        factory(exports);
+    } else {
+        factory((root.rollupTools = {}));
+    }
+}(this, function(exports) {
+
+// =========================================================
+
+function loadModulesRollup(code) {
+    var board = Espruino.Core.Env.getBoardData();
+    var env = Espruino.Core.Env.getData();
+    var modules = [];
+
+    var entryFilename = env.FILE;
+
+    // the env.FILE is only present in the espruino-cli
+    if (!entryFilename) {
+        // the 'modules' contents is written the filesystem in the espruinoRollup()
+        // for in-browser setup with filesystem simulation
+        entryFilename = 'main.js';
+        modules.push([entryFilename, code]);
+    }
+
+    var job = Espruino.Config;
+    var minify = job.MINIFICATION_LEVEL === 'TERSER';
+    var minifyModules = job.MODULE_MINIFICATION_LEVEL === 'TERSER';
+
+    return espruinoRollup.bundle({
+        modules,
+        input: entryFilename,
+        output: {
+            format: 'cjs'
+        },
+        espruino: {
+            job,
+
+            externals: {
+                // for proxy and offline support
+                getURL: url => new Promise((resolve, reject) => {
+                    Espruino.Core.Utils.getURL(url, data => data!==undefined ? resolve(data) : reject(null));
+                }),
+                // for project sandbox chrome app
+                getModule: moduleName => new Promise((resolve, reject) => {
+                    Espruino.callProcessor("getModule",
+                        { moduleName, moduleCode:undefined, isMinified:false },
+                        data => data.moduleCode!==undefined ? resolve(data.moduleCode) : reject(null));
+                })
+            },
+
+            board: board.BOARD ? board : env,
+            mergeModules: job.MODULE_MERGE,
+            minify: minify ? buildEspruinoMinifyOptions() : false,
+            minifyModules
+        }
+    });
+}
+
+function buildEspruinoMinifyOptions() {
+    var job = Espruino.Config;
+
+    var options = {};
+    if (job.MINIFICATION_Mangle === false) {
+        options.mangle = false;
+    }
+    if (job.MINIFICATION_Unused === false) {
+        options.compress = options.compress || {};
+        options.compress.unused = false;
+    }
+    if (job.MINIFICATION_DeadCode === false) {
+        options.compress = options.compress || {};
+        options.compress.dead_code = false;
+    }
+    if (job.MINIFICATION_Unreachable === false) {
+        options.compress = options.compress || {};
+        options.compress.dead_code = false; // in Terser dead_code ~ unreachable
+    }
+    if (job.MINIFICATION_Literal === false) {
+        options.compress = options.compress || {};
+        options.compress.reduce_vars = false;
+    }
+
+    return options;
+}
+
+function minifyCodeTerser(code) {
+    return espruinoRollup.minify(code, buildEspruinoMinifyOptions());
+}
+
+exports.loadModulesRollup = loadModulesRollup
+exports.minifyCodeTerser = minifyCodeTerser;
+
+}));

libs/rollup/index.js

@@ -0,0 +1,26 @@
+{
+  "name": "espruino-rollup",
+  "version": "0.1.0",
+  "description": "Espruino rollup bundling pipeline",
+  "main": "espruino-rollup.js",
+  "scripts": {
+    "build": "rollup -c"
+  },
+  "keywords": [],
+  "author": "Standa Opichal",
+  "license": "MIT",
+  "dependencies": {
+    "memfs": "=2.12.1",
+    "rollup": "^0.67.4",
+    "rollup-plugin-espruino-modules": "opichals/rollup-plugin-espruino-modules"
+  },
+  "devDependencies": {
+    "rollup-plugin-alias": "^1.4.0",
+    "rollup-plugin-commonjs": "^9.1.8",
+    "rollup-plugin-json": "^3.1.0",
+    "rollup-plugin-node-builtins": "^2.1.2",
+    "rollup-plugin-node-globals": "^1.4.0",
+    "rollup-plugin-node-resolve": "^3.4.0",
+    "rollup-plugin-terser": "^3.0.0"
+  }
+}