Use JSZip 3.x to handle zip files

lib/inject.js

@@ -1,97 +1,76 @@
+const BbPromise = require('bluebird');
 const fse = require('fs-extra');
 const glob = require('glob-all');
 const get = require('lodash.get');
 const set = require('lodash.set');
 const path = require('path');
 const values = require('lodash.values');
-const zipper = require('zip-local');
 const JSZip = require('jszip');
+const { writeZip, zipFile } = require('./zipTree');
 
-/**
- * write zip contents to a file
- * @param {Object} zip
- * @param {string} path
- */
-function writeZip(zip, path) {
-  const buff = zip.generate({
-    type: 'nodebuffer',
-    compression: 'DEFLATE'
-  });
-
-  fse.writeFileSync(path, buff);
-}
-
-/**
- * add a new file to a zip file from a buffer
- * @param {Object} zip
- * @param {string} path path to put in zip
- * @param {string} buffer file contents
- */
-function zipFile(zip, path, buffer) {
-  zip.file(path, buffer, {
-    date: new Date(0) // necessary to get the same hash when zipping the same content
-  });
-}
+BbPromise.promisifyAll(fse);
 
 /**
- * inject requirements into packaged application
+ * Inject requirements into packaged application.
  * @param {string} requirementsPath requirements folder path
  * @param {string} packagePath target package path
  * @param {Object} options our options object
+ * @return {Promise} the JSZip object constructed.
  */
 function injectRequirements(requirementsPath, packagePath, options) {
   const noDeploy = new Set(options.noDeploy || []);
 
-  const zip = zipper.sync.unzip(packagePath).lowLevel();
-
-  glob
-    .sync([path.join(requirementsPath, '**')], { mark: true, dot: true })
-    .forEach(file => {
-      if (file.endsWith('/')) {
-        return;
-      }
-
-      const relativeFile = path.relative(requirementsPath, file);
-
-      if (relativeFile.match(/^__pycache__[\\/]/)) {
-        return;
-      }
-      if (noDeploy.has(relativeFile.split(/([-\\/]|\.py$|\.pyc$)/, 1)[0])) {
-        return;
-      }
-
-      zipFile(zip, relativeFile, fse.readFileSync(file));
-    });
-
-  writeZip(zip, packagePath);
+  return fse
+    .readFileAsync(packagePath)
+    .then(buffer => JSZip.loadAsync(buffer))
+    .then(zip =>
+      BbPromise.resolve(
+        glob.sync([path.join(requirementsPath, '**')], {
+          mark: true,
+          dot: true
+        })
+      )
+        .map(file => [file, path.relative(requirementsPath, file)])
+        .filter(
+          ([file, relativeFile]) =>
+            !file.endsWith('/') &&
+            !relativeFile.match(/^__pycache__[\\/]/) &&
+            !noDeploy.has(relativeFile.split(/([-\\/]|\.py$|\.pyc$)/, 1)[0])
+        )
+        .map(([file, relativeFile]) =>
+          zipFile(zip, relativeFile, fse.readFileAsync(file))
+        )
+        .then(() => writeZip(zip, packagePath))
+    );
 }
 
 /**
- * remove all modules but the selected module from a package
- * @param {string} source original package
- * @param {string} target result package
+ * Remove all modules but the selected module from a package.
+ * @param {string} source path to original package
+ * @param {string} target path to result package
  * @param {string} module module to keep
+ * @return {Promise} the JSZip object written out.
  */
 function moveModuleUp(source, target, module) {
-  const sourceZip = zipper.sync.unzip(source).memory();
-  const targetZip = JSZip.make();
-
-  sourceZip.contents().forEach(file => {
-    if (!file.startsWith(module + '/')) {
-      return;
-    }
-    zipFile(
-      targetZip,
-      file.replace(module + '/', ''),
-      sourceZip.read(file, 'buffer')
-    );
-  });
-
-  writeZip(targetZip, target);
+  const targetZip = new JSZip();
+
+  return fse
+    .readFileAsync(source)
+    .then(buffer => JSZip.loadAsync(buffer))
+    .then(sourceZip => sourceZip.filter(file => file.startsWith(module + '/')))
+    .map(srcZipObj =>
+      zipFile(
+        targetZip,
+        srcZipObj.name.replace(module + '/', ''),
+        srcZipObj.async('nodebuffer')
+      )
+    )
+    .then(() => writeZip(targetZip, target));
 }
 
 /**
- * inject requirements into packaged application
+ * Inject requirements into packaged application.
+ * @return {Promise} the combined promise for requirements injection.
  */
 function injectAllRequirements() {
   this.serverless.cli.log('Injecting required Python packages to package...');
@@ -101,30 +80,39 @@ function injectAllRequirements() {
   }
 
   if (this.serverless.service.package.individually) {
-    values(this.serverless.service.functions).forEach(f => {
-      if (
-        !(f.runtime || this.serverless.service.provider.runtime).match(
+    return BbPromise.resolve(values(this.serverless.service.functions))
+      .filter(func =>
+        (func.runtime || this.serverless.service.provider.runtime).match(
           /^python.*/
         )
-      ) {
-        return;
-      }
-      if (!get(f, 'module')) {
-        set(f, ['module'], '.');
-      }
-      if (f.module !== '.') {
-        const artifactPath = path.join('.serverless', `${f.module}.zip`);
-        moveModuleUp(f.package.artifact, artifactPath, f.module);
-        f.package.artifact = artifactPath;
-      }
-      injectRequirements(
-        path.join('.serverless', f.module, 'requirements'),
-        f.package.artifact,
-        this.options
+      )
+      .map(func => {
+        if (!get(func, 'module')) {
+          set(func, ['module'], '.');
+        }
+        return func;
+      })
+      .map(func => {
+        if (func.module !== '.') {
+          const artifact = func.package.artifact;
+          const newArtifact = path.join('.serverless', `${func.module}.zip`);
+          func.package.artifact = newArtifact;
+          return moveModuleUp(artifact, newArtifact, func.module).then(
+            () => func
+          );
+        } else {
+          return func;
+        }
+      })
+      .map(func =>
+        injectRequirements(
+          path.join('.serverless', func.module, 'requirements'),
+          func.package.artifact,
+          this.options
+        )
       );
-    });
   } else {
-    injectRequirements(
+    return injectRequirements(
       path.join('.serverless', 'requirements'),
       this.serverless.service.package.artifact,
       this.options

lib/zip.js

@@ -2,47 +2,43 @@ const fse = require('fs-extra');
 const path = require('path');
 const get = require('lodash.get');
 const set = require('lodash.set');
-const zipper = require('zip-local');
-const BbPromise = require('bluebird');
 const values = require('lodash.values');
+const uniqBy = require('lodash.uniqby');
+const BbPromise = require('bluebird');
+const JSZip = require('jszip');
+const { addTree, writeZip } = require('./zipTree');
 
 BbPromise.promisifyAll(fse);
 
 /**
- * add the vendor helper to the current service tree
+ * Add the vendor helper to the current service tree.
  * @return {Promise}
  */
 function addVendorHelper() {
   if (this.options.zip) {
     if (this.serverless.service.package.individually) {
-      let promises = [];
-      let doneModules = [];
-      values(this.serverless.service.functions).forEach(f => {
-        if (!get(f, 'package.include')) {
-          set(f, ['package', 'include'], []);
-        }
-        if (!get(f, 'module')) {
-          set(f, ['module'], '.');
-        }
-
-        f.package.include.push('unzip_requirements.py');
-
-        if (!doneModules.includes(f.module)) {
+      return BbPromise.resolve(values(this.serverless.service.functions))
+        .map(f => {
+          if (!get(f, 'package.include')) {
+            set(f, ['package', 'include'], []);
+          }
+          if (!get(f, 'module')) {
+            set(f, ['module'], '.');
+          }
+          f.package.include.push('unzip_requirements.py');
+          return f;
+        })
+        .then(functions => uniqBy(functions, func => func.module))
+        .map(f => {
           this.serverless.cli.log(
             `Adding Python requirements helper to ${f.module}...`
           );
 
-          promises.push(
-            fse.copyAsync(
-              path.resolve(__dirname, '../unzip_requirements.py'),
-              path.join(this.servicePath, f.module, 'unzip_requirements.py')
-            )
+          return fse.copyAsync(
+            path.resolve(__dirname, '../unzip_requirements.py'),
+            path.join(this.servicePath, f.module, 'unzip_requirements.py')
           );
-
-          doneModules.push(f.module);
-        }
-      });
-      return BbPromise.all(promises);
+        });
     } else {
       this.serverless.cli.log('Adding Python requirements helper...');
 
@@ -61,31 +57,28 @@ function addVendorHelper() {
 }
 
 /**
- * remove the vendor helper from the current service tree
- * @return {Promise}
+ * Remove the vendor helper from the current service tree.
+ * @return {Promise} the promise to remove the vendor helper.
  */
 function removeVendorHelper() {
   if (this.options.zip && this.options.cleanupZipHelper) {
     if (this.serverless.service.package.individually) {
-      let promises = [];
-      let doneModules = [];
-      values(this.serverless.service.functions).forEach(f => {
-        if (!get(f, 'module')) {
-          set(f, ['module'], '.');
-        }
-        if (!doneModules.includes(f.module)) {
+      return BbPromise.resolve(values(this.serverless.service.functions))
+        .map(f => {
+          if (!get(f, 'module')) {
+            set(f, ['module'], '.');
+          }
+          return f;
+        })
+        .then(funcs => uniqBy(funcs, f => f.module))
+        .map(f => {
           this.serverless.cli.log(
             `Removing Python requirements helper from ${f.module}...`
           );
-          promises.push(
-            fse.removeAsync(
-              path.join(this.servicePath, f.module, 'unzip_requirements.py')
-            )
+          return fse.removeAsync(
+            path.join(this.servicePath, f.module, 'unzip_requirements.py')
           );
-          doneModules.push(f.module);
-        }
-      });
-      return BbPromise.all(promises);
+        });
     } else {
       this.serverless.cli.log('Removing Python requirements helper...');
       return fse.removeAsync(
@@ -96,35 +89,36 @@ function removeVendorHelper() {
 }
 
 /**
- * zip up .serverless/requirements
+ * Zip up .serverless/requirements or .serverless/[MODULE]/requirements.
+ * @return {Promise} the promise to pack requirements.
  */
 function packRequirements() {
   if (this.options.zip) {
     if (this.serverless.service.package.individually) {
-      let doneModules = [];
-      values(this.serverless.service.functions).forEach(f => {
-        if (!get(f, 'module')) {
-          set(f, ['module'], '.');
-        }
-        if (!doneModules.includes(f.module)) {
+      return BbPromise.resolve(values(this.serverless.service.functions))
+        .map(f => {
+          if (!get(f, 'module')) {
+            set(f, ['module'], '.');
+          }
+          return f;
+        })
+        .then(funcs => uniqBy(funcs, f => f.module))
+        .map(f => {
           this.serverless.cli.log(
             `Zipping required Python packages for ${f.module}...`
           );
           f.package.include.push(`${f.module}/.requirements.zip`);
-          zipper.sync
-            .zip(`.serverless/${f.module}/requirements`)
-            .compress()
-            .save(`${f.module}/.requirements.zip`);
-          doneModules.push(f.module);
-        }
-      });
+          return addTree(
+            new JSZip(),
+            `.serverless/${f.module}/requirements`
+          ).then(zip => writeZip(zip, `${f.module}/.requirements.zip`));
+        });
     } else {
       this.serverless.cli.log('Zipping required Python packages...');
       this.serverless.service.package.include.push('.requirements.zip');
-      zipper.sync
-        .zip(path.join(this.servicePath, '.serverless/requirements'))
-        .compress()
-        .save(path.join(this.servicePath, '.requirements.zip'));
+      return addTree(new JSZip(), '.serverless/requirements').then(zip =>
+        writeZip(zip, path.join(this.servicePath, '.requirements.zip'))
+      );
     }
   }
 }