Merge pull request #147 from kichik/inject

Inject files into package instead of creating symlinks

Author: dschep

index.js

@@ -4,9 +4,9 @@
 const BbPromise = require('bluebird');
 const fse = require('fs-extra');
 const {addVendorHelper, removeVendorHelper, packRequirements} = require('./lib/zip');
+const {injectAllRequirements} = require('./lib/inject');
 const {installAllRequirements} = require('./lib/pip');
 const {pipfileToRequirements} = require('./lib/pipenv');
-const {linkAllRequirements, unlinkAllRequirements} = require('./lib/link');
 const {cleanup} = require('./lib/clean');
 
 BbPromise.promisifyAll(fse);
@@ -101,12 +101,11 @@ class ServerlessPythonRequirements {
       .then(pipfileToRequirements)
       .then(addVendorHelper)
       .then(installAllRequirements)
-      .then(packRequirements)
-      .then(linkAllRequirements);
+      .then(packRequirements);
 
     const after = () => BbPromise.bind(this)
       .then(removeVendorHelper)
-      .then(unlinkAllRequirements);
+      .then(injectAllRequirements);
 
     const invalidateCaches = () => {
       if (this.options.invalidateCaches) {

lib/inject.js

@@ -0,0 +1,126 @@
+const fse = require('fs-extra');
+const glob = require('glob-all');
+const get = require('lodash.get');
+const set = require('lodash.set');
+const path = require('path');
+const values = require('lodash.values');
+const zipper = require('zip-local');
+const JSZip = require('jszip');
+
+/**
+ * write zip contents to a file
+ * @param {Object} zip
+ * @param {string} path
+ */
+function writeZip(zip, path) {
+  const buff = zip.generate({
+    type: 'nodebuffer',
+    compression: 'DEFLATE',
+  });
+
+  fse.writeFileSync(path, buff);
+}
+
+/**
+ * add a new file to a zip file from a buffer
+ * @param {Object} zip
+ * @param {string} path path to put in zip
+ * @param {string} buffer file contents
+ */
+function zipFile(zip, path, buffer) {
+  zip.file(path, buffer, {
+    date: new Date(0), // necessary to get the same hash when zipping the same content
+  });
+}
+
+/**
+ * inject requirements into packaged application
+ * @param {string} requirementsPath requirements folder path
+ * @param {string} packagePath target package path
+ * @param {Object} options our options object
+ */
+function injectRequirements(requirementsPath, packagePath, options) {
+  const noDeploy = new Set(options.noDeploy || []);
+
+  const zip = zipper.sync.unzip(packagePath).lowLevel();
+
+  glob.sync([path.join(requirementsPath, '**')], {mark: true, dot: true}).forEach((file) => {
+    if (file.endsWith('/')) {
+      return;
+    }
+
+    const relativeFile = path.relative(requirementsPath, file);
+
+    if (relativeFile.match(/^__pycache__[\\\/]/)) {
+      return;
+    }
+    if (noDeploy.has(relativeFile.split(/([-\\\/]|\.py$|\.pyc$)/, 1)[0])) {
+      return;
+    }
+
+    zipFile(zip, relativeFile, fse.readFileSync(file));
+  });
+
+  writeZip(zip, packagePath);
+}
+
+/**
+ * remove all modules but the selected module from a package
+ * @param {string} source original package
+ * @param {string} target result package
+ * @param {string} module module to keep
+ */
+function moveModuleUp(source, target, module) {
+  const sourceZip = zipper.sync.unzip(source).memory();
+  const targetZip = JSZip.make();
+
+  sourceZip.contents().forEach((file) => {
+    if (!file.startsWith(module + '/')) {
+      return;
+    }
+    zipFile(targetZip, file.replace(module + '/', ''), sourceZip.read(file, 'buffer'));
+  });
+
+  writeZip(targetZip, target);
+}
+
+/**
+ * inject requirements into packaged application
+ */
+function injectAllRequirements() {
+  this.serverless.cli.log('Injecting required Python packages to package...');
+
+  if (this.options.zip) {
+    return;
+  }
+
+  if (this.serverless.service.package.individually) {
+    values(this.serverless.service.functions)
+      .forEach((f) => {
+        if (!(f.runtime || this.serverless.service.provider.runtime).match(/^python.*/)) {
+          return;
+        }
+        if (!get(f, 'module')) {
+          set(f, ['module'], '.');
+        }
+        if (f.module !== '.') {
+          const artifactPath = path.join('.serverless', `${f.module}.zip`);
+          moveModuleUp(f.package.artifact, artifactPath, f.module);
+          f.package.artifact = artifactPath;
+        }
+        injectRequirements(
+          path.join('.serverless', f.module, 'requirements'),
+          f.package.artifact,
+          this.options
+        );
+      });
+  } else {
+    injectRequirements(
+      path.join('.serverless', 'requirements'),
+      this.serverless.service.package.artifact,
+      this.options
+    );
+  }
+}
+
+module.exports = {injectAllRequirements};

lib/link.js

@@ -48,11 +48,12 @@
   "dependencies": {
     "bluebird": "^3.0.6",
     "fs-extra": "^3.0.1",
+    "glob-all": "^3.1.0",
     "is-wsl": "^1.1.0",
+    "jszip": "^2.5.0",
     "lodash.get": "^4.4.2",
     "lodash.set": "^4.3.2",
     "lodash.values": "^4.3.0",
-    "rimraf": "^2.6.2",
     "shell-quote": "^1.6.1",
     "zip-local": "^0.3.4"
   }

package.json

@@ -27,6 +27,7 @@ teardown() {
     sls --zip=true package
     unzip .serverless/sls-py-req-test.zip -d puck
     ls puck/.requirements.zip puck/unzip_requirements.py
+    ! ls puck/flask
 }
 
 @test "py3.6 doesn't package boto3 by default" {
@@ -40,6 +41,7 @@ teardown() {
     sls package
     unzip .serverless/sls-py-req-test.zip -d puck
     ! ls puck/bottle.py
+    ! ls puck/__pycache__/bottle.cpython-36.pyc
 }
 
 @test "py3.6 can package flask with zip & dockerizePip option" {