add a zip method that decompresses! (and remove options) closes #3

Removes zipimport functionality because it does not work
with most libraries you want to use it with (ones with shared
objects). And remove non-linking non-zip way of using it. This
simplifies the usage options and the code base. win-win.

Author: dschep

README.md

@@ -22,66 +22,39 @@ plugins:
 ```
 
 
-## Adding the dependencies to `sys.path`
-
-### Automatic
-The default behavior of this plugin is to link libraries into the working tree
-during deployment so that they are in your handler's `PYTHONPATH` when running
-on lambda.
-
-### Manual
-This method is required when using [ZipImport](#zipimport) support and can be
-enabled manually by adding the following option to your config:
-
-```yaml
-custom:
-  pythonRequirements:
-    link: false
-```
-
-`serverless-python-requirements` adds a module called `requirements` to your
-puck. To easily make the bundled dependencies available, simply import it. Eg.
-add this to the top of any file using dependencies specified in your
-`requirements.txt`:
-```python
-import requirements
-# Now you can use deps you specified in requirements.txt!
-import requests
-```
-
 ## Cross compiling!
-Compiling non-pure-Python modules is supported on MacOS via the use of Docker
-and the [docker-lambda](https://github.com/lambci/docker-lambda) image.
+Compiling non-pure-Python modules or fetching their manylinux wheels is
+supported on non-linux OSs via the use of Docker and the
+[docker-lambda](https://github.com/lambci/docker-lambda) image.
 To enable docker usage, add the following to your `serverless.yml`:
 ```yaml
 custom:
   pythonRequirements:
     dockerizePip: true
 ```
 
-## ZipImport!
+## Dealing with Lambda's size limitations
 To help deal with potentially large dependencies (for example: `numpy`, `scipy`
-and `scikit-learn`) there is support for having python import using
-[zipimport](https://docs.python.org/2.7/library/zipimport.html). To enable this
-add the following to your  `serverless.yml`:
+and `scikit-learn`) there is support for compressing the libraries. This does
+require a minor change to your code to decompress them.  To enable this add the
+following to your  `serverless.yml`:
 ```yaml
 custom:
   pythonRequirements:
-    zipImport: true
+    zip: true
 ```
 
-
-## Limitations
- * if using the `package` directive in `serverless.yml` ensure that
-`requirements.py` is are included as well as `.requirements` or
-`.requirements.zip` if using [ZipImport](#zipimport).
-
+and add this to your handler module before any code that imports your deps:
+```python
+import unzip_requirements
+```
 
 ## Manual invocations
 
-The `.requirements` and `requirements.py` files are left behind to simplify
-development. To clean them up, run `sls requirements clean`. You can also
-install them manually for local development with `sls requirements install`.
+The `.requirements` and `requirements.zip`(if using zip support) files are left
+behind to speed things up on subsequent deploys. To clean them up, run
+`sls requirements clean`. You can also create them manually with
+`sls requirements install`.
 
 ## Credit
 This plugin is influenced by

index.js

@@ -11,12 +11,21 @@ const {zipDirectory} = require('./zipService');
 BbPromise.promisifyAll(fse);
 
 class ServerlessPythonRequirements {
-  packVendorHelper() {
-    this.serverless.cli.log('Installing Python requirements helper...');
+  addVendorHelper() {
+    if (this.custom.zip) {
+      this.serverless.cli.log('Removing Python requirements helper...');
 
-    return fse.copyAsync(
-      path.resolve(__dirname, 'requirements.py'),
-      path.join(this.serverless.config.servicePath, 'requirements.py'));
+      return fse.copyAsync(
+        path.resolve(__dirname, 'unzip_requirements.py'),
+        path.join(this.serverless.config.servicePath, 'unzip_requirements.py'));
+    }
+  };
+
+  removeVendorHelper() {
+    if (this.custom.zip) {
+      this.serverless.cli.log('Adding Python requirements helper...');
+      return fse.removeAsync('unzip_requirements.py');
+    }
   };
 
   installRequirements() {
@@ -60,32 +69,34 @@ class ServerlessPythonRequirements {
 
   packRequirements() {
     return this.installRequirements().then(() => {
-      if (this.custom.zipImport)
+      if (this.custom.zip) {
+        this.serverless.cli.log('Zipping required Python packages...');
         return zipDirectory('.requirements', '.requirements.zip');
+      }
     });
   }
 
   linkRequirements() {
-    if (!this.custom.zipImport && this.custom.link) {
+    if (!this.custom.zip) {
       this.serverless.cli.log('Linking required Python packages...');
       fse.readdirSync('.requirements').map(file =>
         fse.symlinkSync(`.requirements/${file}`, `./${file}`));
     }
   }
 
   unlinkRequirements() {
-    if (!this.custom.zipImport && this.custom.link) {
+    if (!this.custom.zip) {
       this.serverless.cli.log('Unlinking required Python packages...');
       fse.readdirSync('.requirements').map(file => fse.unlinkSync(file));
     }
   }
 
   cleanup() {
-    const artifacts = ['requirements.py'];
-    if (this.custom.zipImport)
-      artifacts.push('.requirements.zip')
-    else
-      artifacts.push('.requirements')
+    const artifacts = ['.requirements'];
+    if (this.custom.zip) {
+      artifacts.push('.requirements.zip');
+      artifacts.push('unzip_requirements.py');
+    }
 
     return BbPromise.all(_.map(artifacts, (artifact) =>
       fse.removeAsync(path.join(this.serverless.config.servicePath, artifact))));;
@@ -95,16 +106,19 @@ class ServerlessPythonRequirements {
     this.serverless = serverless;
     this.options = options;
     this.custom = Object.assign({
-      zipImport: false,
-      link: true,
+      zip: false,
     }, this.serverless.service.custom &&
     this.serverless.service.custom.pythonRequirements || {});
 
+    if (!_.has(this.serverless.service, ['package', 'exclude']))
+      _.set(this.serverless.service, ['package', 'exclude'], []);
+    this.serverless.service.package.exclude.push('.requirements/**');
+
     this.commands = {
       'requirements': {
         commands: {
           'clean': {
-            usage: 'Remove .requirements and requirements.py',
+            usage: 'Remove .requirements and requirements.zip',
             lifecycleEvents: [
               'clean',
             ],
@@ -121,15 +135,16 @@ class ServerlessPythonRequirements {
 
     this.hooks = {
       'before:deploy:createDeploymentArtifacts': () => BbPromise.bind(this)
-        .then(this.packVendorHelper)
+        .then(this.addVendorHelper)
         .then(this.packRequirements)
         .then(this.linkRequirements),
 
       'after:deploy:createDeploymentArtifacts': () => BbPromise.bind(this)
+        .then(this.removeVendorHelper)
         .then(this.unlinkRequirements),
 
       'requirements:install:install': () => BbPromise.bind(this)
-        .then(this.packVendorHelper)
+        .then(this.addVendorHelper)
         .then(this.packRequirements),
 
       'requirements:clean:clean': () => BbPromise.bind(this)

package.json

@@ -1,6 +1,6 @@
 {
   "name": "serverless-python-requirements",
-  "version": "2.0.0-beta.3",
+  "version": "2.0.0-beta.4",
   "engines": {
     "node": ">=4.0"
   },
@@ -32,7 +32,7 @@
     "zipService.js",
     "LICENSE",
     "package.json",
-    "requirements.py",
+    "unzip_requirements.py",
     "README.md"
   ],
   "main": "index.js",

requirements.py

@@ -0,0 +1,14 @@
+import os
+import sys
+import zipfile
+import tempfile
+
+
+zip_requirements = os.path.join(
+    os.path.split(__file__)[0], '.requirements.zip')
+
+tempdir = tempfile.mkdtemp()
+
+sys.path.append(tempdir)
+
+zipfile.ZipFile(zip_requirements, 'r').extractall(tempdir)