Merge pull request #4 from UnitedIncome/feature/zipimport

Use zipped libs to deal with aws size limits

Author: dschep

README.md

@@ -40,11 +40,26 @@ and the [docker-lambda](https://github.com/lambci/docker-lambda) image.
 To enable docker usage, add the following to your `serverless.yml`:
 ```yaml
 custom:
-  dockerizePip: true
+  pythonRequirements:
+    dockerizePip: true
 ```
 
+## ZipImport!
+To help deal with potentially large dependencies (for example: `numpy`, `scipy`
+and `scikit-learn`) there is support for having python import using
+[zipimport](https://docs.python.org/2.7/library/zipimport.html). To enable this
+add the following to your  `serverless.yml`:
+```yaml
+custom:
+  pythonRequirements:
+    zipImport: true
+```
+
+
 ## Limitations
- * if using the `package` directive in `serverless.yml` ensure that `.requirements` and `requirements.py` are included.
+ * if using the `package` directive in `serverless.yml` ensure that
+`requirements.py` is are included as well as `.requirements` or
+`.requirements.zip` if using [ZipImport](#zipimport).
 
 
 ## Manual invocations

index.js

@@ -6,6 +6,7 @@ const _ = require('lodash');
 const path = require('path');
 const fse = require('fs-extra');
 const child_process = require('child_process');
+const Zip = require('adm-zip');
 
 BbPromise.promisifyAll(fse);
 
@@ -18,7 +19,7 @@ class ServerlessPythonRequirements {
       path.join(this.serverless.config.servicePath, 'requirements.py'));
   };
 
-  packRequirements() {
+  installRequirements() {
     if (!fse.existsSync(path.join(this.serverless.config.servicePath, 'requirements.txt'))) {
       return BbPromise.resolve();
     }
@@ -32,8 +33,7 @@ class ServerlessPythonRequirements {
         '-t', '.requirements',
         '-r', 'requirements.txt',
       ];
-      if (this.serverless.service.custom &&
-          this.serverless.service.custom.dockerizePip) {
+      if (this.custom.dockerizePip) {
         cmd = 'docker';
         options = [
           'run', '--rm',
@@ -52,8 +52,25 @@ class ServerlessPythonRequirements {
     });
   };
 
+  packRequirements() {
+    return this.installRequirements().then(() => {
+      return new BbPromise((resolve, reject) => {
+        if (this.custom.zipImport) {
+          const zip = new Zip();
+          zip.addLocalFolder('.requirements', '');
+          zip.writeZip('.requirements.zip');
+          fse.remove('.requirements', (err) => err?reject():resolve());
+        } else resolve();
+      });
+    });
+  }
+
   cleanup() {
-    const artifacts = ['requirements.py', '.requirements'];
+    const artifacts = ['requirements.py'];
+    if (this.custom.zipImport)
+      artifacts.push('.requirements.zip')
+    else
+      artifacts.push('.requirements')
 
     return BbPromise.all(_.map(artifacts, (artifact) =>
       fse.removeAsync(path.join(this.serverless.config.servicePath, artifact))));;
@@ -76,6 +93,7 @@ class ServerlessPythonRequirements {
   constructor(serverless, options) {
     this.serverless = serverless;
     this.options = options;
+    this.custom = this.serverless.service.custom && this.serverless.service.custom.pythonRequirements || {};
 
     this.commands = {
       'requirements': {

package.json

@@ -40,6 +40,7 @@
   "devDependencies": {},
   "dependencies": {
     "bluebird": "^3.0.6",
+    "adm-zip": "0.4.7",
     "fs-extra": "^0.26.7",
     "lodash": "^4.13.1"
   }

requirements.py

@@ -3,9 +3,12 @@
 
 
 requirements = os.path.join(
-    os.path.split(__file__)[0],
-    '.requirements',
-)
+    os.path.split(__file__)[0], '.requirements')
+zip_requirements = os.path.join(
+    os.path.split(__file__)[0], '.requirements.zip')
 
-if requirements  not in sys.path:
+if requirements not in sys.path:
     sys.path.append(requirements)
+
+if zip_requirements not in sys.path:
+    sys.path.append(zip_requirements)