Merge branch 'master' into docker_windows_paths

Author: dschep

.editorconfig

@@ -0,0 +1,16 @@
+root = true
+
+[*]
+end_of_line = lf
+insert_final_newline = true
+
+[*.{js,json,yml}]
+charset = utf-8
+
+[*.js]
+indent_style = space
+indent_size = 2
+
+[package.json]
+indent_style = space
+indent_size = 2

.gitattributes

@@ -0,0 +1,2 @@
+*.js    text eol=lf
+*.json  text eol=lf

README.md

@@ -119,6 +119,14 @@ custom:
           - .requirements-cache
 ```
 
+When using `--cache-dir` don't forget to also exclude it from the package.
+
+```yaml
+package:
+  exclude:
+    - .requirements-cache/**
+```
+
 ### Customize requirements file name
 [Some `pip` workflows involve using requirements files not named
 `requirements.txt`](https://www.kennethreitz.org/essays/a-better-pip-workflow).

circle.yml

@@ -13,10 +13,12 @@ jobs:
       - run: python3.6 -m pip install pipenv
       # install nodejs
       - run: curl -sL https://deb.nodesource.com/setup_6.x | bash - && apt -y install nodejs
-      # install serverless
-      - run: npm install -g serverless
+      # install serverless & depcheck
+      - run: npm install -g serverless depcheck
       # install deps
       - run: npm i
+      # depcheck
+      - run: depcheck .
       # lint:
       - run: npm run lint
       # test!

lib/docker.js

@@ -18,7 +18,7 @@ function dockerCommand(options) {
     throw new Error(ps.stderr);
   }
   return ps;
-}
+};
 
 /**
  * Build the custom Docker image
@@ -34,34 +34,67 @@ function buildImage(dockerFile) {
   return imageName;
 };
 
+/**
+ * Test bind path to make sure it's working
+ * @param {string} bindPath
+ * @return {boolean}
+ */
+function tryBindPath(bindPath) {
+  const options = ['run', '--rm', '-v', `${bindPath}:/test`,
+    'alpine', 'ls', '/test/serverless.yml'];
+  try {
+    const ps = dockerCommand(options);
+    return ps.stdout.trim() === '/test/serverless.yml';
+  } catch (err) {
+    return false;
+  }
+};
+
 /**
  * Get bind path depending on os platform
  * @param {string} servicePath
  * @return {string} The bind path.
  */
 function getBindPath(servicePath) {
-  // Determine os platform of docker CLI from 'docker version'
-  const options = ['version', '--format', '{{with .Client}}{{.Os}}{{end}}'];
-  const ps = dockerCommand(options);
-  const cliPlatform = ps.stdout.trim();
-
   // Determine bind path
-  let bindPath;
-  if (process.platform === 'win32') {
-    bindPath = servicePath.replace(/\\([^\s])/g, '/$1');
-    if (cliPlatform === 'windows') {
-      bindPath = bindPath.replace(/^\/(\w)\//i, '$1:/');
-    }
-  } else if (isWsl) {
-    bindPath = servicePath.replace(/^\/mnt\//, '/');
-    if (cliPlatform === 'windows') {
-      bindPath = bindPath.replace(/^\/(\w)\//i, '$1:/');
-    }
+  if (process.platform !== 'win32' && !isWsl) {
+    return servicePath;
+  }
+
+  let bindPaths = [];
+  let baseBindPath = servicePath.replace(/\\([^\s])/g, '/$1');
+  let drive;
+  let path;
+
+  bindPaths.push(baseBindPath);
+  if (baseBindPath.startsWith('/mnt/')) { // cygwin "/mnt/C/users/..."
+    baseBindPath = baseBindPath.replace(/^\/mnt\//, '/');
+  }
+  if (baseBindPath[1] == ':') { // normal windows "c:/users/..."
+    drive = baseBindPath[0];
+    path = baseBindPath.substring(3);
+  } else if (baseBindPath[0] == '/' && baseBindPath[2] == '/') { // gitbash "/c/users/..."
+    drive = baseBindPath[1];
+    path = baseBindPath.substring(3);
   } else {
-    bindPath = servicePath;
+    throw new Error(`Unknown path format ${baseBindPath.substr(10)}...`);
+  }
+
+  bindPaths.push(`/${drive.toLowerCase()}/${path}`);
+  bindPaths.push(`/${drive.toUpperCase()}/${path}`);
+  bindPaths.push(`/mnt/${drive.toLowerCase()}/${path}`);
+  bindPaths.push(`/mnt/${drive.toUpperCase()}/${path}`);
+  bindPaths.push(`${drive.toLowerCase()}:/${path}`);
+  bindPaths.push(`${drive.toUpperCase()}:/${path}`);
+
+  for (let i = 0; i < bindPaths.length; i++) {
+    const bindPath = bindPaths[i];
+    if (tryBindPath(bindPath)) {
+      return bindPath;
+    }
   }
 
-  return bindPath;
+  throw new Error('Unable to find good bind path format');
 };
 
 module.exports = {buildImage, getBindPath};

lib/link.js

@@ -16,7 +16,7 @@ function makeSymlink(source, target) {
   try {
     fse.symlinkSync(source, target);
   } catch (exception) {
-    if (exception.code === 'EPERM' && process.platform !== 'win32') {
+    if (exception.code === 'EPERM' && process.platform === 'win32') {
       fse.copySync(source, target);
     } else {
       let linkDest = null;
@@ -36,14 +36,14 @@ function makeSymlink(source, target) {
  * @param {string} sourceFolder
  * @param {string} targetFolder
  * @param {boolean} isPython
- * @param {Object} package
+ * @param {Object} packageObj
  * @param {Object} serverless
  * @param {string} servicePath
  * @param {Object} options
  * @return {undefined}
  */
 function linkRequirements(
-  sourceFolder, targetFolder, isPython, package, serverless, servicePath, options
+  sourceFolder, targetFolder, isPython, packageObj, serverless, servicePath, options
 ) {
   const requirementsDir = path.join(servicePath, sourceFolder);
   if (fse.existsSync('__pycache__')) {
@@ -59,12 +59,12 @@ function linkRequirements(
 
       // don't include python deps in non-python functions
       if (isPython) {
-        if (!package.exclude.includes(file)) {
-          package.include.push(file);
-          package.include.push(`${file}/**`);
+        if (!packageObj.exclude.includes(file)) {
+          packageObj.include.push(file);
+          packageObj.include.push(`${file}/**`);
           makeSymlink(`${requirementsDir}/${file}`, `${targetFolder}/${file}`);
         } else {
-          package.exclude.push(`${file}/**`);
+          packageObj.exclude.push(`${file}/**`);
         }
       }
     });

lib/pip.js

@@ -22,26 +22,20 @@ function installRequirements(requirementsPath, targetFolder, serverless, service
   fse.ensureDirSync(targetRequirementsFolder);
 
   const dotSlsReqs = path.join(targetFolder, 'requirements.txt');
-  let fileName = requirementsPath;
   if (options.usePipenv && fse.existsSync(path.join(servicePath, 'Pipfile'))) {
-    fileName = dotSlsReqs;
+    generateRequirementsFile(dotSlsReqs, dotSlsReqs, options);
+  } else {
+    generateRequirementsFile(requirementsPath, dotSlsReqs, options);
   }
 
   serverless.cli.log(`Installing requirements of ${requirementsPath} in ${targetFolder}...`);
 
-  // In case the requirements file is a symlink, copy it to targetFolder
-  // if using docker to avoid errors
-  if (options.dockerizePip && fileName !== dotSlsReqs) {
-    fse.copySync(fileName, dotSlsReqs);
-    fileName = dotSlsReqs;
-  }
-
   let cmd;
   let cmdOptions;
   let pipCmd = [
     options.pythonBin, '-m', 'pip', '--isolated', 'install',
     '-t', dockerPathForWin(options, targetRequirementsFolder),
-    '-r', dockerPathForWin(options, fileName),
+    '-r', dockerPathForWin(options, dotSlsReqs),
     ...options.pipCmdExtraArgs,
   ];
   if (!options.dockerizePip) {
@@ -136,6 +130,20 @@ function dockerPathForWin(options, path) {
   return path;
 }
 
+/** create a filtered requirements.txt without anything from noDeploy
+ * @param {string} source requirements
+ * @param {string} target requirements where results are written
+ * @param {Object} options
+ */
+function generateRequirementsFile(source, target, options) {
+  const noDeploy = new Set(options.noDeploy || []);
+  const requirements = fse.readFileSync(source, {encoding: 'utf-8'}).split(/\r?\n/);
+  const filteredRequirements = requirements.filter((req) => {
+    return !noDeploy.has(req.split(/[=<> \t]/)[0].trim());
+  });
+  fse.writeFileSync(target, filteredRequirements.join('\n'));
+}
+
 /**
  * pip install the requirements to the .serverless/requirements directory
  * @return {undefined}

package.json

@@ -46,11 +46,8 @@
     "eslint-config-google": "^0.7.1"
   },
   "dependencies": {
-    "archiver": "^1.3.0",
     "bluebird": "^3.0.6",
     "fs-extra": "^3.0.1",
-    "glob-all": "^3.1.0",
-    "graceful-fs": "^4.1.11",
     "is-wsl": "^1.1.0",
     "lodash.get": "^4.4.2",
     "lodash.set": "^4.3.2",

unzip_requirements.py

@@ -1,14 +1,20 @@
 import os
+import shutil
 import sys
 import zipfile
 
 
-zip_requirements = os.path.join(
-    os.environ.get('LAMBDA_TASK_ROOT', os.getcwd()), '.requirements.zip')
+pkgdir = '/tmp/sls-py-req'
 
-tempdir = '/tmp/sls-py-req'
+sys.path.append(pkgdir)
 
-sys.path.append(tempdir)
+if not os.path.exists(pkgdir):
+    tempdir = '/tmp/_temp-sls-py-req'
+    if os.path.exists(tempdir):
+        shutil.rmtree(tempdir)
+
+    zip_requirements = os.path.join(
+        os.environ.get('LAMBDA_TASK_ROOT', os.getcwd()), '.requirements.zip')
 
-if not os.path.exists(tempdir):
     zipfile.ZipFile(zip_requirements, 'r').extractall(tempdir)
+    os.rename(tempdir, pkgdir)  # Atomic