Add support for poetry.

squaresurf · squaresurf · commit 60a389545536 · 2019-02-04T08:43:58.000-08:00
Why: * [poetry](https://github.com/sdispater/poetry) is a Python dependency management and packaging tool. This change addresses the need by: * Taking the same approach as pipenv. * Install poetry on CircleCI and AppVeyor for tests. * Use chmodSync in test.js as it works better cross platform for setting exact permissions on files. See: nodejs/node#1104 Side effects: * There may be conflicts or oddities if there is a Pipfile and a pyproject.toml.
diff --git a/.tool-versions b/.tool-versions
@@ -0,0 +1,2 @@
+nodejs 6.16.0
+python 3.6.8 2.7.15
diff --git a/README.md b/README.md
@@ -94,6 +94,23 @@ custom:
 ```
 
 
+## Poetry support :sparkles::pencil::sparkles:
+NOTE: Only poetry version 1 supports the required `export` command for this
+feature. As of the point this feature was added, poetry 1.0.0 was in preview
+and requires that poetry is installed with the --preview flag.
+
+TL;DR Install poetry with the `--preview` flag.
+
+If you include a `pyproject.toml` and have `poetry` installed instead of a `requirements.txt` this will use
+`poetry export --without-hashes -f requirements.txt` to generate them. It is fully compatible with all options such as `zip` and
+`dockerizePip`. If you don't want this plugin to generate it for you, set the following option:
+```yaml
+custom:
+  pythonRequirements:
+    usePoetry: false
+```
+
+
 ## Dealing with Lambda's size limitations
 To help deal with potentially large dependencies (for example: `numpy`, `scipy`
 and `scikit-learn`) there is support for compressing the libraries. This does
@@ -405,3 +422,4 @@ zipinfo .serverless/xxx.zip
  * [@alexjurkiewicz](https://github.com/alexjurkiewicz) - [docs about docker workflows](#native-code-dependencies-during-build)
  * [@andrewfarley](https://github.com/andrewfarley) - Implemented download caching and static caching
  * [@bweigel](https://github.com/bweigel) - adding the `slimPatternsAppendDefaults` option & fixing per-function packaging when some functions don't have requirements & Porting tests from bats to js!
+ * [@squaresurf](https://github.com/squaresurf) - adding usePoetry option
diff --git a/appveyor.yml b/appveyor.yml
@@ -1,6 +1,7 @@
 version: '{build}'
 init:
     - cmd: pip install pipenv
+    - cmd: pip install poetry==1.0.0a2
     - ps: npm i -g serverless
 build: off
 test_script:
diff --git a/circle.yml b/circle.yml
@@ -25,6 +25,11 @@ jobs:
       - run: sudo apt -y update && sudo apt -y install python-pip python2.7 curl unzip
       # instal pipenv
       - run: sudo python3.6 -m pip install pipenv pip-tools
+      # install poetry
+      - run: |
+          curl https://raw.githubusercontent.com/sdispater/poetry/master/get-poetry.py -o get-poetry.py
+          python get-poetry.py --preview --yes
+          rm get-poetry.py
       # install nodejs
       - run: curl -sL https://deb.nodesource.com/setup_6.x | sudo bash - && sudo apt -y install nodejs
       # install serverless & depcheck
@@ -36,4 +41,8 @@ jobs:
       # lint:
       - run: npm run lint
       # test!
-      - run: npm run test
+      - run: |
+          export PATH="$HOME/.poetry/bin:$PATH"
+          export LC_ALL=C.UTF-8
+          export LANG=C.UTF-8
+          npm run test
diff --git a/index.js b/index.js
@@ -12,6 +12,7 @@ const {
 const { injectAllRequirements } = require('./lib/inject');
 const { installAllRequirements } = require('./lib/pip');
 const { pipfileToRequirements } = require('./lib/pipenv');
+const { pyprojectTomlToRequirements } = require('./lib/poetry');
 const { cleanup, cleanupCache } = require('./lib/clean');
 
 BbPromise.promisifyAll(fse);
@@ -35,6 +36,7 @@ class ServerlessPythonRequirements {
         invalidateCaches: false,
         fileName: 'requirements.txt',
         usePipenv: true,
+        usePoetry: true,
         pythonBin:
           process.platform === 'win32'
             ? 'python.exe'
@@ -156,6 +158,7 @@ class ServerlessPythonRequirements {
       }
       return BbPromise.bind(this)
         .then(pipfileToRequirements)
+        .then(pyprojectTomlToRequirements)
         .then(addVendorHelper)
         .then(installAllRequirements)
         .then(packRequirements);
diff --git a/lib/pip.js b/lib/pip.js
@@ -60,7 +60,22 @@ function generateRequirementsFile(
   servicePath,
   options
 ) {
-  if (options.usePipenv && fse.existsSync(path.join(servicePath, 'Pipfile'))) {
+  if (
+    options.usePoetry &&
+    fse.existsSync(path.join(servicePath, 'pyproject.toml'))
+  ) {
+    filterRequirementsFile(
+      path.join(servicePath, '.serverless/requirements.txt'),
+      targetFile,
+      options
+    );
+    serverless.cli.log(
+      `Parsed requirements.txt from pyproject.toml in ${targetFile}...`
+    );
+  } else if (
+    options.usePipenv &&
+    fse.existsSync(path.join(servicePath, 'Pipfile'))
+  ) {
     filterRequirementsFile(
       path.join(servicePath, '.serverless/requirements.txt'),
       targetFile,
@@ -377,6 +392,31 @@ function copyVendors(vendorFolder, targetFolder, serverless) {
   });
 }
 
+/**
+ * This checks if requirements file exists.
+ * @param {string} servicePath
+ * @param {Object} options
+ * @param {string} fileName
+ */
+function requirementsFileExists(servicePath, options, fileName) {
+  if (
+    options.usePoetry &&
+    fse.existsSync(path.join(servicePath, 'pyproject.toml'))
+  ) {
+    return true;
+  }
+
+  if (options.usePipenv && fse.existsSync(path.join(servicePath, 'Pipfile'))) {
+    return true;
+  }
+
+  if (fse.existsSync(fileName)) {
+    return true;
+  }
+
+  return false;
+}
+
 /**
  * This evaluates if requirements are actually needed to be installed, but fails
  * gracefully if no req file is found intentionally.  It also assists with code
@@ -399,17 +439,8 @@ function installRequirementsIfNeeded(
   const fileName = path.join(servicePath, modulePath, options.fileName);
 
   // Skip requirements generation, if requirements file doesn't exist
-  if (options.usePipenv) {
-    if (
-      !fse.existsSync(path.join(servicePath, 'Pipfile')) &&
-      !fse.existsSync(fileName)
-    ) {
-      return false;
-    }
-  } else {
-    if (!fse.existsSync(fileName)) {
-      return false;
-    }
+  if (!requirementsFileExists(servicePath, options, fileName)) {
+    return false;
   }
 
   let requirementsTxtDirectory;
diff --git a/lib/poetry.js b/lib/poetry.js
@@ -0,0 +1,43 @@
+const fse = require('fs-extra');
+const path = require('path');
+const { spawnSync } = require('child_process');
+
+/**
+ * poetry install
+ */
+function pyprojectTomlToRequirements() {
+  if (
+    !this.options.usePoetry ||
+    !fse.existsSync(path.join(this.servicePath, 'pyproject.toml'))
+  ) {
+    return;
+  }
+
+  this.serverless.cli.log('Generating requirements.txt from pyproject.toml...');
+
+  const res = spawnSync(
+    'poetry',
+    ['export', '--without-hashes', '-f', 'requirements.txt'],
+    {
+      cwd: this.servicePath
+    }
+  );
+  if (res.error) {
+    if (res.error.code === 'ENOENT') {
+      throw new Error(
+        `poetry not found! Install it according to the poetry docs.`
+      );
+    }
+    throw new Error(res.error);
+  }
+  if (res.status !== 0) {
+    throw new Error(res.stderr);
+  }
+  fse.ensureDirSync(path.join(this.servicePath, '.serverless'));
+  fse.moveSync(
+    path.join(this.servicePath, 'requirements.txt'),
+    path.join(this.servicePath, '.serverless', 'requirements.txt')
+  );
+}
+
+module.exports = { pyprojectTomlToRequirements };
diff --git a/test.js b/test.js
@@ -4,6 +4,7 @@ const glob = require('glob-all');
 const JSZip = require('jszip');
 const tape = require('tape');
 const {
+  chmodSync,
   removeSync,
   readFileSync,
   copySync,
@@ -710,6 +711,94 @@ test("pipenv py3.6 doesn't package bottle with noDeploy option", t => {
   t.end();
 });
 
+test('poetry py3.6 can package flask with default options', t => {
+  process.chdir('tests/poetry');
+  const path = npm(['pack', '../..']);
+  npm(['i', path]);
+  sls(['package']);
+  const zipfiles = listZipFiles('.serverless/sls-py-req-test.zip');
+  t.true(zipfiles.includes(`flask${sep}__init__.py`), 'flask is packaged');
+  t.false(zipfiles.includes(`boto3${sep}__init__.py`), 'boto3 is NOT packaged');
+  t.end();
+});
+
+test('poetry py3.6 can package flask with slim option', t => {
+  process.chdir('tests/poetry');
+  const path = npm(['pack', '../..']);
+  npm(['i', path]);
+  sls(['--slim=true', 'package']);
+  const zipfiles = listZipFiles('.serverless/sls-py-req-test.zip');
+  t.true(zipfiles.includes(`flask${sep}__init__.py`), 'flask is packaged');
+  t.deepEqual(
+    zipfiles.filter(filename => filename.endsWith('.pyc')),
+    [],
+    'no pyc files packaged'
+  );
+  t.true(
+    zipfiles.filter(filename => filename.endsWith('__main__.py')).length > 0,
+    '__main__.py files are packaged'
+  );
+  t.end();
+});
+
+test('poetry py3.6 can package flask with slim & slimPatterns options', t => {
+  process.chdir('tests/poetry');
+
+  copySync('_slimPatterns.yml', 'slimPatterns.yml');
+  const path = npm(['pack', '../..']);
+  npm(['i', path]);
+  sls(['--slim=true', 'package']);
+  const zipfiles = listZipFiles('.serverless/sls-py-req-test.zip');
+  t.true(zipfiles.includes(`flask${sep}__init__.py`), 'flask is packaged');
+  t.deepEqual(
+    zipfiles.filter(filename => filename.endsWith('.pyc')),
+    [],
+    'no pyc files packaged'
+  );
+  t.deepEqual(
+    zipfiles.filter(filename => filename.endsWith('__main__.py')),
+    [],
+    '__main__.py files are NOT packaged'
+  );
+  t.end();
+});
+
+test('poetry py3.6 can package flask with zip option', t => {
+  process.chdir('tests/poetry');
+  const path = npm(['pack', '../..']);
+  npm(['i', path]);
+  sls([`--pythonBin=${getPythonBin(3)}`, '--zip=true', 'package']);
+  const zipfiles = listZipFiles('.serverless/sls-py-req-test.zip');
+  t.true(
+    zipfiles.includes('.requirements.zip'),
+    'zipped requirements are packaged'
+  );
+  t.true(zipfiles.includes(`unzip_requirements.py`), 'unzip util is packaged');
+  t.false(
+    zipfiles.includes(`flask${sep}__init__.py`),
+    "flask isn't packaged on its own"
+  );
+  t.end();
+});
+
+test("poetry py3.6 doesn't package bottle with noDeploy option", t => {
+  process.chdir('tests/poetry');
+  const path = npm(['pack', '../..']);
+  npm(['i', path]);
+  perl([
+    '-p',
+    '-i.bak',
+    '-e',
+    's/(pythonRequirements:$)/\\1\\n    noDeploy: [bottle]/',
+    'serverless.yml'
+  ]);
+  sls(['package']);
+  const zipfiles = listZipFiles('.serverless/sls-py-req-test.zip');
+  t.true(zipfiles.includes(`flask${sep}__init__.py`), 'flask is packaged');
+  t.false(zipfiles.includes(`bottle.py`), 'bottle is NOT packaged');
+  t.end();
+});
+
 test('py3.6 can package flask with zip option and no explicit include', t => {
   process.chdir('tests/base');
   const path = npm(['pack', '../..']);
@@ -760,7 +849,8 @@ test(
       's/(handler.py.*$)/$1\n    - foobar/',
       'serverless.yml'
     ]);
-    writeFileSync(`foobar`, '', { mode: perm });
+    writeFileSync(`foobar`, '');
+    chmodSync(`foobar`, perm);
     sls(['--vendor=./vendor', 'package']);
 
     const zipfiles = listZipFiles('.serverless/sls-py-req-test.zip');
@@ -1001,6 +1091,27 @@ test('pipenv py3.6 can package flask with slim & slimPatterns & slimPatternsAppe
   t.end();
 });
 
+test('poetry py3.6 can package flask with slim & slimPatterns & slimPatternsAppendDefaults=false  option', t => {
+  process.chdir('tests/poetry');
+  copySync('_slimPatterns.yml', 'slimPatterns.yml');
+  const path = npm(['pack', '../..']);
+  npm(['i', path]);
+
+  sls(['--slim=true', '--slimPatternsAppendDefaults=false', 'package']);
+  const zipfiles = listZipFiles('.serverless/sls-py-req-test.zip');
+  t.true(zipfiles.includes(`flask${sep}__init__.py`), 'flask is packaged');
+  t.true(
+    zipfiles.filter(filename => filename.endsWith('.pyc')).length >= 1,
+    'pyc files are packaged'
+  );
+  t.deepEqual(
+    zipfiles.filter(filename => filename.endsWith('__main__.py')),
+    [],
+    '__main__.py files are NOT packaged'
+  );
+  t.end();
+});
+
 test('py3.6 can package flask with package individually option', t => {
   process.chdir('tests/base');
   const path = npm(['pack', '../..']);
@@ -1449,7 +1560,8 @@ test(
     process.chdir('tests/individually');
     const path = npm(['pack', '../..']);
     const perm = '775';
-    writeFileSync(`module1${sep}foobar`, '', { mode: perm });
+    writeFileSync(`module1${sep}foobar`, '');
+    chmodSync(`module1${sep}foobar`, perm);
 
     npm(['i', path]);
     sls(['package']);
@@ -1475,6 +1587,7 @@ test(
     const path = npm(['pack', '../..']);
     const perm = '775';
     writeFileSync(`module1${sep}foobar`, '', { mode: perm });
+    chmodSync(`module1${sep}foobar`, perm);
 
     npm(['i', path]);
     sls(['--dockerizePip=true', 'package']);
diff --git a/tests/poetry/.gitignore b/tests/poetry/.gitignore
@@ -0,0 +1,22 @@
+# Distribution / packaging
+.Python
+env/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# Serverless
+.serverless
+.requirements
+unzip_requirements.py
diff --git a/tests/poetry/_slimPatterns.yml b/tests/poetry/_slimPatterns.yml
@@ -0,0 +1,2 @@
+slimPatterns:
+  - "**/__main__.py"
diff --git a/tests/poetry/handler.py b/tests/poetry/handler.py
@@ -0,0 +1,5 @@
+import requests
+
+
+def hello(event, context):
+    return requests.get('https://httpbin.org/get').json()
diff --git a/tests/poetry/package.json b/tests/poetry/package.json
diff --git a/tests/poetry/poetry.lock b/tests/poetry/poetry.lock
diff --git a/tests/poetry/pyproject.toml b/tests/poetry/pyproject.toml
diff --git a/tests/poetry/serverless.yml b/tests/poetry/serverless.yml

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,2 @@`
	`1`	`+nodejs 6.16.0`
	`2`	`+python 3.6.8 2.7.15`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,2 @@`
	`1`	`+slimPatterns:`
	`2`	`+ - "**/__main__.py"`