Merge pull request serverless#507 from jacksgt/pr-editable

Handle all requirement flags properly and strip -e flags

Author: bsamuel-ui

README.md

@@ -163,6 +163,18 @@ except ImportError:
   pass
 ```
 
+## Dealing with Lambda's size limitations #2
+
+If you don't want to zip your requirements, you can define a list of patterns in the serverless config using the `excluded`
+option and glob syntax
+
+```yaml
+custom:
+  pythonRequirements:
+    excluded:
+      - "Crypto/SelfTest"
+```
+
 ### Slim Package
 
 _Works on non 'win32' environments: Docker, WSL are included_

index.js

@@ -15,6 +15,7 @@ const { installAllRequirements } = require('./lib/pip');
 const { pipfileToRequirements } = require('./lib/pipenv');
 const { pyprojectTomlToRequirements } = require('./lib/poetry');
 const { cleanup, cleanupCache } = require('./lib/clean');
+const { removeExcluded } = require('./lib/exclude');
 
 BbPromise.promisifyAll(fse);
 
@@ -55,6 +56,7 @@ class ServerlessPythonRequirements {
         useDownloadCache: true,
         cacheLocation: false,
         staticCacheMaxVersions: 0,
+        excluded: [],
         pipCmdExtraArgs: [],
         noDeploy: [],
         vendor: ''
@@ -173,6 +175,7 @@ class ServerlessPythonRequirements {
         .then(pyprojectTomlToRequirements)
         .then(addVendorHelper)
         .then(installAllRequirements)
+        .then(removeExcluded)
         .then(packRequirements)
         .then(setupArtifactPathCapturing);
     };

lib/exclude.js

@@ -0,0 +1,29 @@
+const path = require('path');
+const glob = require('glob-all');
+const fse = require('fs-extra');
+
+/**
+ * remove all excluded patterns
+ * @return {undefined}
+ */
+function removeExcluded() {
+  if (Array.isArray(this.options.excluded) && this.options.excluded.length > 0) { 
+    const DEBUG = process.env.SLS_DEBUG;
+
+    const basePath = path.join(
+        this.servicePath,
+        '.serverless',
+        `requirements`
+    );
+    if (DEBUG) this.serverless.cli.log(`Performing removal of excluded files`);
+    for (const pattern of this.options.excluded) {
+      if (DEBUG) this.serverless.cli.log(`Pattern: ${basePath}/${pattern}`);
+      for (const file of glob.sync(`${basePath}/${pattern}`)) {
+          if (DEBUG) this.serverless.cli(`Found, removing: ${file}`);
+          fse.removeSync(file);
+      }
+    }
+  }
+}
+
+module.exports = { removeExcluded };

lib/pip.js

@@ -69,7 +69,8 @@ function generateRequirementsFile(
     filterRequirementsFile(
       path.join(servicePath, '.serverless/requirements.txt'),
       targetFile,
-      options
+      options,
+      serverless
     );
     serverless.cli.log(
       `Parsed requirements.txt from pyproject.toml in ${targetFile}...`
@@ -81,13 +82,14 @@ function generateRequirementsFile(
     filterRequirementsFile(
       path.join(servicePath, '.serverless/requirements.txt'),
       targetFile,
-      options
+      options,
+      serverless
     );
     serverless.cli.log(
       `Parsed requirements.txt from Pipfile in ${targetFile}...`
     );
   } else {
-    filterRequirementsFile(requirementsPath, targetFile, options);
+    filterRequirementsFile(requirementsPath, targetFile, options, serverless);
     serverless.cli.log(
       `Generated requirements from ${requirementsPath} in ${targetFile}...`
     );
@@ -372,13 +374,13 @@ function getRequirements(source) {
  *  assist with matching the static cache.  The sorting will skip any
  *  lines starting with -- as those are typically ordered at the
  *  start of a file ( eg: --index-url / --extra-index-url ) or any
- *  lines that start with -f or -i,  Please see:
+ *  lines that start with -c, -e, -f, -i or -r,  Please see:
  * https://pip.pypa.io/en/stable/reference/pip_install/#requirements-file-format
  * @param {string} source requirements
  * @param {string} target requirements where results are written
  * @param {Object} options
  */
-function filterRequirementsFile(source, target, options) {
+function filterRequirementsFile(source, target, options, serverless) {
   const noDeploy = new Set(options.noDeploy || []);
   const requirements = getRequirements(source);
   var prepend = [];
@@ -389,10 +391,23 @@ function filterRequirementsFile(source, target, options) {
       return false;
     } else if (
       req.startsWith('--') ||
+      req.startsWith('-c') ||
+      req.startsWith('-e') ||
       req.startsWith('-f') ||
-      req.startsWith('-i')
+      req.startsWith('-i') ||
+      req.startsWith('-r')
     ) {
-      // If we have options (prefixed with --) keep them for later
+      if (req.startsWith('-e')) {
+        // strip out editable flags
+        // not required inside final archive and avoids pip bugs
+        // see https://github.com/UnitedIncome/serverless-python-requirements/issues/240
+        req = req.split('-e')[1].trim();
+        serverless.cli.log(
+          `Warning: Stripping -e flag from requirement ${req}`
+        );
+      }
+
+      // Keep options for later
       prepend.push(req);
       return false;
     } else if (req === '') {

test.js

@@ -376,6 +376,28 @@ test(
   { skip: !hasPython(3) }
 );
 
+test(
+  'py3.6 can package boto3 with editable',
+  async t => {
+    process.chdir('tests/base');
+    const path = npm(['pack', '../..']);
+    npm(['i', path]);
+    sls([
+      `--pythonBin=${getPythonBin(3)}`,
+      '--fileName=requirements-w-editable.txt',
+      'package'
+    ]);
+    const zipfiles = await listZipFiles('.serverless/sls-py-req-test.zip');
+    t.true(zipfiles.includes(`boto3${sep}__init__.py`), 'boto3 is packaged');
+    t.true(
+      zipfiles.includes(`botocore${sep}__init__.py`),
+      'botocore is packaged'
+    );
+    t.end();
+  },
+  { skip: !hasPython(3) }
+);
+
 test(
   'py3.6 can package flask with dockerizePip option',
   async t => {

tests/base/requirements-w-editable.txt

@@ -0,0 +1 @@
+-e git+https://github.com/boto/boto3.git#egg=boto3