From 59dcb77f4875d2dff40a7fb530afc4304e6a1ef6 Mon Sep 17 00:00:00 2001 From: Daniel Schep Date: Tue, 3 Jan 2017 18:43:52 -0500 Subject: [PATCH 1/4] First pass at using zipped libs to deal with aws size limits. fixes #3 --- index.js | 29 +++++++++++++++++++++++++++-- package.json | 1 + requirements.py | 11 +++++++---- 3 files changed, 35 insertions(+), 6 deletions(-) diff --git a/index.js b/index.js index c826bb88..6cda4edd 100644 --- a/index.js +++ b/index.js @@ -6,6 +6,7 @@ const _ = require('lodash'); const path = require('path'); const fse = require('fs-extra'); const child_process = require('child_process'); +const {EasyZip} = require('easy-zip'); BbPromise.promisifyAll(fse); @@ -18,7 +19,7 @@ class ServerlessPythonRequirements { path.join(this.serverless.config.servicePath, 'requirements.py')); }; - packRequirements() { + installRequirements() { if (!fse.existsSync(path.join(this.serverless.config.servicePath, 'requirements.txt'))) { return BbPromise.resolve(); } @@ -52,8 +53,32 @@ class ServerlessPythonRequirements { }); }; + packRequirements() { + return this.installRequirements().then(() => { + return new BbPromise((resolve, reject) => { + if (this.serverless.service.custom && this.serverless.service.custom.zipImport) { + const zip = new EasyZip(); + zip.zipFolder('.requirements', (err) => { + if (err) { + reject(); + return; + } + zip.writeToFile('.requirements.zip'); + fse.remove('.requirements', (err) => err?reject():resolve()); + }, {rootFolder: '.'}); + } else { + resolve(); + } + }); + }); + } + cleanup() { - const artifacts = ['requirements.py', '.requirements']; + const artifacts = ['requirements.py']; + if (this.serverless.service.custom && this.serverless.service.custom.zipImport) + artifacts.push('.requirements.zip') + else + artifacts.push('.requirements') return BbPromise.all(_.map(artifacts, (artifact) => fse.removeAsync(path.join(this.serverless.config.servicePath, artifact))));; diff --git a/package.json b/package.json index 5bc8c627..2d793f5e 100644 --- a/package.json +++ b/package.json @@ -40,6 +40,7 @@ "devDependencies": {}, "dependencies": { "bluebird": "^3.0.6", + "easy-zip": "github:owenchong/easy-zip", "fs-extra": "^0.26.7", "lodash": "^4.13.1" } diff --git a/requirements.py b/requirements.py index 69b62859..182e25fa 100644 --- a/requirements.py +++ b/requirements.py @@ -3,9 +3,12 @@ requirements = os.path.join( - os.path.split(__file__)[0], - '.requirements', -) + os.path.split(__file__)[0], '.requirements') +zip_requirements = os.path.join( + os.path.split(__file__)[0], '.requirements.zip') -if requirements not in sys.path: +if requirements not in sys.path: sys.path.append(requirements) + +if zip_requirements not in sys.path: + sys.path.append(zip_requirements) From bb949e9f43582fd39735126f96056de597e47fe9 Mon Sep 17 00:00:00 2001 From: Daniel Schep Date: Tue, 3 Jan 2017 19:06:12 -0500 Subject: [PATCH 2/4] document ZipImport --- README.md | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index eead5dfa..f64f29d4 100644 --- a/README.md +++ b/README.md @@ -41,8 +41,21 @@ custom: dockerizePip: true ``` +## ZipImport! +To help deal with potentially large dependencies (for example: `numpy`, `scipy` +and `scikit-learn`) there is support for having python import using +[zipimport](https://docs.python.org/2.7/library/zipimport.html). To enable this +add the following to your `serverless.yml`: +```yaml +custom: + zipImport: true +``` + + ## Limitations - * if using the `package` directive in `serverless.yml` ensure that `.requirements` and `requirements.py` are included. + * if using the `package` directive in `serverless.yml` ensure that +`requirements.py` is are included as well as `.requirements` or +`.requirements.zip` if using [ZipImport](#zipimport). ## Manual invocations From 4ca40f24638d6f3dfb18843caf58fb3a951f2a8a Mon Sep 17 00:00:00 2001 From: Daniel Schep Date: Wed, 4 Jan 2017 12:58:04 -0500 Subject: [PATCH 3/4] Switch to adm-zip! --- index.js | 19 ++++++------------- package.json | 2 +- 2 files changed, 7 insertions(+), 14 deletions(-) diff --git a/index.js b/index.js index 6cda4edd..150d612d 100644 --- a/index.js +++ b/index.js @@ -6,7 +6,7 @@ const _ = require('lodash'); const path = require('path'); const fse = require('fs-extra'); const child_process = require('child_process'); -const {EasyZip} = require('easy-zip'); +const Zip = require('adm-zip'); BbPromise.promisifyAll(fse); @@ -57,18 +57,11 @@ class ServerlessPythonRequirements { return this.installRequirements().then(() => { return new BbPromise((resolve, reject) => { if (this.serverless.service.custom && this.serverless.service.custom.zipImport) { - const zip = new EasyZip(); - zip.zipFolder('.requirements', (err) => { - if (err) { - reject(); - return; - } - zip.writeToFile('.requirements.zip'); - fse.remove('.requirements', (err) => err?reject():resolve()); - }, {rootFolder: '.'}); - } else { - resolve(); - } + const zip = new Zip(); + zip.addLocalFolder('.requirements', ''); + zip.writeZip('.requirements.zip'); + fse.remove('.requirements', (err) => err?reject():resolve()); + } else resolve(); }); }); } diff --git a/package.json b/package.json index 2d793f5e..ca27b399 100644 --- a/package.json +++ b/package.json @@ -40,7 +40,7 @@ "devDependencies": {}, "dependencies": { "bluebird": "^3.0.6", - "easy-zip": "github:owenchong/easy-zip", + "adm-zip": "0.4.7", "fs-extra": "^0.26.7", "lodash": "^4.13.1" } From dc01040f993b4c79fac991a20c03f4af8795ce0f Mon Sep 17 00:00:00 2001 From: Daniel Schep Date: Wed, 4 Jan 2017 13:10:31 -0500 Subject: [PATCH 4/4] group options together in custom --- README.md | 6 ++++-- index.js | 8 ++++---- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index f64f29d4..0c980ddb 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,8 @@ and the [docker-lambda](https://github.com/lambci/docker-lambda) image. To enable docker usage, add the following to your `serverless.yml`: ```yaml custom: - dockerizePip: true + pythonRequirements: + dockerizePip: true ``` ## ZipImport! @@ -48,7 +49,8 @@ and `scikit-learn`) there is support for having python import using add the following to your `serverless.yml`: ```yaml custom: - zipImport: true + pythonRequirements: + zipImport: true ``` diff --git a/index.js b/index.js index 150d612d..e0e5323c 100644 --- a/index.js +++ b/index.js @@ -33,8 +33,7 @@ class ServerlessPythonRequirements { '-t', '.requirements', '-r', 'requirements.txt', ]; - if (this.serverless.service.custom && - this.serverless.service.custom.dockerizePip) { + if (this.custom.dockerizePip) { cmd = 'docker'; options = [ 'run', '--rm', @@ -56,7 +55,7 @@ class ServerlessPythonRequirements { packRequirements() { return this.installRequirements().then(() => { return new BbPromise((resolve, reject) => { - if (this.serverless.service.custom && this.serverless.service.custom.zipImport) { + if (this.custom.zipImport) { const zip = new Zip(); zip.addLocalFolder('.requirements', ''); zip.writeZip('.requirements.zip'); @@ -68,7 +67,7 @@ class ServerlessPythonRequirements { cleanup() { const artifacts = ['requirements.py']; - if (this.serverless.service.custom && this.serverless.service.custom.zipImport) + if (this.custom.zipImport) artifacts.push('.requirements.zip') else artifacts.push('.requirements') @@ -94,6 +93,7 @@ class ServerlessPythonRequirements { constructor(serverless, options) { this.serverless = serverless; this.options = options; + this.custom = this.serverless.service.custom && this.serverless.service.custom.pythonRequirements || {}; this.commands = { 'requirements': {