From 0931088d017dc4fb8914856edb9b9d03b254a560 Mon Sep 17 00:00:00 2001 From: cgrimal Date: Wed, 17 Jan 2018 17:07:21 +0100 Subject: [PATCH 1/4] Add the dockerFile option --- README.md | 8 ++++++++ index.js | 11 ++++++++++- lib/pip.js | 29 +++++++++++++++++++++++++++-- 3 files changed, 45 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 08e71b25..08bfac22 100644 --- a/README.md +++ b/README.md @@ -40,6 +40,14 @@ custom: ``` This must be the full image name and tag to use, including the runtime specific tag if applicable. +Alternatively, you can define your Docker image in your own Dockerfile and add the following to your `serverless.yml`: +```yaml +custom: + pythonRequirements: + dockerFile: Dockerfile +``` +With `Dockerfile` the name of the Dockerfile that MUST be in your current directory. +Please note the `dockerImage` and the `dockerFile` are mutually exclusive. To install requirements from private git repositories, add the following to your `serverless.yml`: ```yaml diff --git a/index.js b/index.js index 3cdad945..ddf6fe22 100644 --- a/index.js +++ b/index.js @@ -28,7 +28,9 @@ class ServerlessPythonRequirements { usePipenv: true, pythonBin: this.serverless.service.provider.runtime || 'python', dockerizePip: false, - dockerImage: `lambci/lambda:build-${this.serverless.service.provider.runtime}`, + dockerSsh: false, + dockerImage: null, + dockerFile: null, pipCmdExtraArgs: [], noDeploy: [ 'boto3', @@ -45,6 +47,13 @@ class ServerlessPythonRequirements { if (options.dockerizePip === 'non-linux') { options.dockerizePip = process.platform !== 'linux'; } + if (options.dockerImage && options.dockerFile) { + throw new Error('You can provide a dockerImage or a dockerFile option, not both.'); + } else if (!options.dockerFile) { + // If no dockerFile is provided, use default image + const defaultImage = `lambci/lambda:build-${this.serverless.service.provider.runtime}`; + options.dockerImage = options.dockerImage || defaultImage; + } return options; } diff --git a/lib/pip.js b/lib/pip.js index 47319428..31c273eb 100644 --- a/lib/pip.js +++ b/lib/pip.js @@ -53,7 +53,32 @@ function installRequirements() { if (this.options.dockerizePip) { cmd = 'docker'; - this.serverless.cli.log(`Docker Image: ${this.options.dockerImage}`); + let dockerImage = this.options.dockerImage; + if (this.options.dockerFile) { + this.serverless.cli.log(`Building custom docker image from ${this.options.dockerFile}...`); + // Build docker image from provided Dockerfile + const imageName = 'sls-py-reqs-custom'; + options = [ + 'build', '-f', this.options.dockerFile, '-t', imageName, '.' + ]; + const ps = spawnSync(cmd, options, { + 'timeout': 10000, + 'encoding': 'utf-8' + }); + if (ps.error) { + if (ps.error.code === 'ENOENT') { + throw new Error('docker not found! Please install it.'); + } + throw new Error(ps.error); + } else if (ps.status !== 0) { + throw new Error(ps.stderr); + } + + // Set dockerImage option value + dockerImage = imageName; + } + + this.serverless.cli.log(`Docker Image: ${dockerImage}`); // Determine os platform of docker CLI from 'docker version' options = ['version', '--format', '{{with .Client}}{{.Os}}{{end}}']; @@ -103,7 +128,7 @@ function installRequirements() { ]); pipCmd = ['/bin/bash', '-c', '"' + pipCmd + ' && ' + chownCmd + '"']; } - options.push(this.options.dockerImage); + options.push(dockerImage); options.push(...pipCmd); } else { cmd = pipCmd[0]; From e47b2ab8bddb1288bd24559bac36a91ee7dfee0c Mon Sep 17 00:00:00 2001 From: cgrimal Date: Thu, 18 Jan 2018 09:40:26 +0100 Subject: [PATCH 2/4] Fix documentation on path to the Dockerfile --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 08bfac22..e8040e23 100644 --- a/README.md +++ b/README.md @@ -44,9 +44,9 @@ Alternatively, you can define your Docker image in your own Dockerfile and add t ```yaml custom: pythonRequirements: - dockerFile: Dockerfile + dockerFile: ./path/to/Dockerfile ``` -With `Dockerfile` the name of the Dockerfile that MUST be in your current directory. +With `Dockerfile` the path to the Dockerfile that must be in the current folder (or a subfolder). Please note the `dockerImage` and the `dockerFile` are mutually exclusive. To install requirements from private git repositories, add the following to your `serverless.yml`: From 8de19f52a8c94036830a4568ea6998eb1b64d06d Mon Sep 17 00:00:00 2001 From: cgrimal Date: Thu, 18 Jan 2018 10:05:37 +0100 Subject: [PATCH 3/4] Add warning for docker related options --- index.js | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/index.js b/index.js index ddf6fe22..f44779a0 100644 --- a/index.js +++ b/index.js @@ -48,12 +48,22 @@ class ServerlessPythonRequirements { options.dockerizePip = process.platform !== 'linux'; } if (options.dockerImage && options.dockerFile) { - throw new Error('You can provide a dockerImage or a dockerFile option, not both.'); + throw new Error( + 'Python Requirements: you can provide a dockerImage or a dockerFile option, not both.' + ); } else if (!options.dockerFile) { // If no dockerFile is provided, use default image const defaultImage = `lambci/lambda:build-${this.serverless.service.provider.runtime}`; options.dockerImage = options.dockerImage || defaultImage; } + if (!options.dockerizePip && (options.dockerSsh || options.dockerImage || options.dockerFile)) { + if (!this.warningLogged) { + this.serverless.cli.log( + 'WARNING: You provided a docker related option but dockerizePip is set to false.' + ); + this.warningLogged = true; + } + } return options; } @@ -66,6 +76,7 @@ class ServerlessPythonRequirements { constructor(serverless, options) { this.serverless = serverless; this.servicePath = this.serverless.config.servicePath; + this.warningLogged = false; this.commands = { requirements: { From 9ffdf6fafc2d5559ae41911bb8aaf3e96cae9557 Mon Sep 17 00:00:00 2001 From: cgrimal Date: Thu, 18 Jan 2018 11:20:57 +0100 Subject: [PATCH 4/4] Move some docker logic to a docker.js module --- lib/docker.js | 59 +++++++++++++++++++++++++++++++++++++++++++++++++++ lib/pip.js | 58 +++++++------------------------------------------- 2 files changed, 67 insertions(+), 50 deletions(-) create mode 100644 lib/docker.js diff --git a/lib/docker.js b/lib/docker.js new file mode 100644 index 00000000..c62747c7 --- /dev/null +++ b/lib/docker.js @@ -0,0 +1,59 @@ +const {spawnSync} = require('child_process'); +const isWsl = require('is-wsl'); + + +function dockerCommand(options) { + const cmd = 'docker'; + const ps = spawnSync(cmd, options, {'timeout': 10000, 'encoding': 'utf-8'}); + if (ps.error) { + if (ps.error.code === 'ENOENT') { + throw new Error('docker not found! Please install it.'); + } + throw new Error(ps.error); + } else if (ps.status !== 0) { + throw new Error(ps.stderr); + } + return ps; +} + +/** + * Build the custom Docker image + */ +function buildImage(dockerFile) { + const imageName = 'sls-py-reqs-custom'; + const options = [ + 'build', '-f', dockerFile, '-t', imageName, '.' + ]; + const ps = dockerCommand(options); + return imageName; +}; + +/** + * Get bind path depending on os platform + */ +function getBindPath(servicePath) { + // Determine os platform of docker CLI from 'docker version' + const options = ['version', '--format', '{{with .Client}}{{.Os}}{{end}}']; + const ps = dockerCommand(options); + const cliPlatform = ps.stdout.trim(); + + // Determine bind path + let bindPath; + if (process.platform === 'win32') { + bindPath = servicePath.replace(/\\([^\s])/g, '/$1'); + if (cliPlatform === 'windows') { + bindPath = bindPath.replace(/^\/(\w)\//i, '$1:/'); + } + } else if (isWsl) { + bindPath = servicePath.replace(/^\/mnt\//, '/'); + if (cliPlatform === 'windows') { + bindPath = bindPath.replace(/^\/(\w)\//i, '$1:/'); + } + } else { + bindPath = servicePath; + } + + return bindPath; +}; + +module.exports = {buildImage, getBindPath}; diff --git a/lib/pip.js b/lib/pip.js index 31c273eb..62e5bb02 100644 --- a/lib/pip.js +++ b/lib/pip.js @@ -1,8 +1,8 @@ const fse = require('fs-extra'); const path = require('path'); const {spawnSync} = require('child_process'); -const isWsl = require('is-wsl'); const {quote} = require('shell-quote'); +const {buildImage, getBindPath} = require('./docker'); /** * pip install the requirements to the .serverless/requirements directory @@ -53,60 +53,18 @@ function installRequirements() { if (this.options.dockerizePip) { cmd = 'docker'; - let dockerImage = this.options.dockerImage; + // Build docker image if required + let dockerImage; if (this.options.dockerFile) { this.serverless.cli.log(`Building custom docker image from ${this.options.dockerFile}...`); - // Build docker image from provided Dockerfile - const imageName = 'sls-py-reqs-custom'; - options = [ - 'build', '-f', this.options.dockerFile, '-t', imageName, '.' - ]; - const ps = spawnSync(cmd, options, { - 'timeout': 10000, - 'encoding': 'utf-8' - }); - if (ps.error) { - if (ps.error.code === 'ENOENT') { - throw new Error('docker not found! Please install it.'); - } - throw new Error(ps.error); - } else if (ps.status !== 0) { - throw new Error(ps.stderr); - } - - // Set dockerImage option value - dockerImage = imageName; + dockerImage = buildImage(this.options.dockerFile); + } else { + dockerImage = this.options.dockerImage; } - this.serverless.cli.log(`Docker Image: ${dockerImage}`); - // Determine os platform of docker CLI from 'docker version' - options = ['version', '--format', '{{with .Client}}{{.Os}}{{end}}']; - const ps = spawnSync(cmd, options, {'timeout': 10000, 'encoding': 'utf-8'}); - if (ps.error) { - if (ps.error.code === 'ENOENT') { - throw new Error('docker not found! Please install it.'); - } - throw new Error(ps.error); - } else if (ps.status !== 0) { - throw new Error(ps.stderr); - } - - let bindPath; - const cliPlatform = ps.stdout.trim(); - if (process.platform === 'win32') { - bindPath = this.servicePath.replace(/\\([^\s])/g, '/$1'); - if (cliPlatform === 'windows') { - bindPath = bindPath.replace(/^\/(\w)\//i, '$1:/'); - } - } else if (isWsl) { - bindPath = this.servicePath.replace(/^\/mnt\//, '/'); - if (cliPlatform === 'windows') { - bindPath = bindPath.replace(/^\/(\w)\//i, '$1:/'); - } - } else { - bindPath = this.servicePath; - } + // Prepare bind path depending on os platform + const bindPath = getBindPath(this.servicePath); options = [ 'run', '--rm',