pip and inject, I think merged good

Author: Farley

lib/inject.js

@@ -7,6 +7,7 @@ const path = require('path');
 const values = require('lodash.values');
 const JSZip = require('jszip');
 const { writeZip, zipFile } = require('./zipTree');
+const {getRequirementsWorkingPath, md5Path} = require('./shared');
 
 BbPromise.promisifyAll(fse);
 
@@ -115,8 +116,13 @@ function injectAllRequirements(funcArtifact) {
         )
       );
   } else {
+    const reqsFilePath = path.join('.serverless', 'requirements.txt');
+    const reqChecksum = md5Path(reqsFilePath);
+    const workingReqsFolder = getRequirementsWorkingPath(
+      reqChecksum, this.servicePath, this.options);
+
     return injectRequirements(
-      path.join('.serverless', 'requirements'),
+      workingReqsFolder,
       this.serverless.service.package.artifact || funcArtifact,
       this.options
     );

lib/pip.js

@@ -6,37 +6,47 @@ const set = require('lodash.set');
 const { spawnSync } = require('child_process');
 const values = require('lodash.values');
 const { buildImage, getBindPath, getDockerUid } = require('./docker');
+const {
+  checkForAndDeleteMaxCacheVersions, md5Path,
+  getRequirementsWorkingPath, getUserCachePath,
+} = require('./shared');
 
 /**
- * Install requirements described in requirementsPath to targetFolder
+ * Just generate the requirements file in the .serverless folder
  * @param {string} requirementsPath
- * @param {string} targetFolder
+ * @param {string} targetFile
  * @param {Object} serverless
  * @param {string} servicePath
  * @param {Object} options
  * @return {undefined}
  */
-function installRequirements(
+function installRequirementsFile(
   requirementsPath,
-  targetFolder,
+  targetFile,
   serverless,
   servicePath,
   options
 ) {
-  // Create target folder if it does not exist
-  const targetRequirementsFolder = path.join(targetFolder, 'requirements');
-  fse.ensureDirSync(targetRequirementsFolder);
-
-  const dotSlsReqs = path.join(targetFolder, 'requirements.txt');
   if (options.usePipenv && fse.existsSync(path.join(servicePath, 'Pipfile'))) {
-    generateRequirementsFile(dotSlsReqs, dotSlsReqs, options);
+    generateRequirementsFile(dotSlsReqs, targetFile, options);
+    serverless.cli.log(`Parsed requirements.txt from Pipfile in ${targetFile}...`);
   } else {
-    generateRequirementsFile(requirementsPath, dotSlsReqs, options);
+    generateRequirementsFile(requirementsPath, targetFile, options);
+    serverless.cli.log(`Generated requirements from ${requirementsPath} in ${targetFile}...`);
   }
+}
 
-  serverless.cli.log(
-    `Installing requirements of ${requirementsPath} in ${targetFolder}...`
-  );
+/**
+ * Install requirements described from requirements in the targetFolder into that same targetFolder
+ * @param {string} targetFolder
+ * @param {Object} serverless
+ * @param {Object} options
+ * @return {undefined}
+ */
+function installRequirements(targetFolder, serverless, options) {
+  const targetRequirementsTxt = path.join(targetFolder, 'requirements.txt');
+
+  serverless.cli.log(`Installing requirements from ${targetRequirementsTxt} ...`);
 
   let cmd;
   let cmdOptions;
@@ -45,13 +55,20 @@ function installRequirements(
     '-m',
     'pip',
     'install',
-    '-t',
-    dockerPathForWin(options, targetRequirementsFolder),
-    '-r',
-    dockerPathForWin(options, dotSlsReqs),
     ...options.pipCmdExtraArgs
   ];
   if (!options.dockerizePip) {
+    // Push our local OS-specific paths for requirements and target directory
+    pipCmd.push('-t', targetFolder);
+    pipCmd.push('-r', targetRequirementsTxt);
+    // If we want a download cache...
+    if (options.useDownloadCache) {
+      const downloadCacheDir = path.join(getUserCachePath(options), 'downloadCacheslspyc');
+      serverless.cli.log(`Using download cache directory ${downloadCacheDir}`);
+      fse.ensureDirSync(downloadCacheDir);
+      pipCmd.push('--cache-dir', downloadCacheDir);
+    }
+    
     // Check if pip has Debian's --system option and set it if so
     const pipTestRes = spawnSync(options.pythonBin, [
       '-m',
@@ -71,8 +88,13 @@ function installRequirements(
       pipCmd.push('--system');
     }
   }
+  // If we are dockerizing pip
   if (options.dockerizePip) {
     cmd = 'docker';
+    
+    // Push docker-specific paths for requirements and target directory
+    pipCmd.push('-t', '/var/task/');
+    pipCmd.push('-r', '/var/task/requirements.txt');
 
     // Build docker image if required
     let dockerImage;
@@ -87,7 +109,7 @@ function installRequirements(
     serverless.cli.log(`Docker Image: ${dockerImage}`);
 
     // Prepare bind path depending on os platform
-    const bindPath = getBindPath(servicePath);
+    const bindPath = getBindPath(targetFolder);
 
     cmdOptions = ['run', '--rm', '-v', `"${bindPath}:/var/task:z"`];
     if (options.dockerSsh) {
@@ -103,11 +125,22 @@ function installRequirements(
       cmdOptions.push('-v', `${process.env.SSH_AUTH_SOCK}:/tmp/ssh_sock:z`);
       cmdOptions.push('-e', 'SSH_AUTH_SOCK=/tmp/ssh_sock');
     }
+
+    // If we want a download cache...
+    if (options.useDownloadCache) {
+      const downloadCacheDir = path.join(getUserCachePath(options), 'downloadCacheslspyc');
+      serverless.cli.log(`Using download cache directory ${downloadCacheDir}`);
+      fse.ensureDirSync(downloadCacheDir);
+      // And now push it to a volume mount and to pip...
+      cmdOptions.push('-v', `${downloadCacheDir}:/var/useDownloadCache:z`);
+      pipCmd.push('--cache-dir', '/var/useDownloadCache');
+    }
+
     if (process.platform === 'linux') {
       // Use same user so requirements folder is not root and so --cache-dir works
       cmdOptions.push('-u', `${process.getuid()}`);
       // const stripCmd = quote([
-      //   'find', targetRequirementsFolder,
+      //   'find', targetFolder,
       //   '-name', '"*.so"',
       //   '-exec', 'strip', '{}', '\;',
       // ]);
@@ -122,7 +155,7 @@ function installRequirements(
     cmd = pipCmd[0];
     cmdOptions = pipCmd.slice(1);
   }
-  const res = spawnSync(cmd, cmdOptions, { cwd: servicePath, shell: true });
+  const res = spawnSync(cmd, cmdOptions, { cwd: targetFolder, shell: true });
   if (res.error) {
     if (res.error.code === 'ENOENT') {
       if (options.dockerizePip) {
@@ -139,20 +172,9 @@ function installRequirements(
   }
 }
 
-/**
- * convert path from Windows style to Linux style, if needed
- * @param {Object} options
- * @param {string} path
- * @return {string}
- */
-function dockerPathForWin(options, path) {
-  if (process.platform === 'win32' && options.dockerizePip) {
-    return path.replace('\\', '/');
-  }
-  return path;
-}
-
 /** create a filtered requirements.txt without anything from noDeploy
+ *  then remove all comments and empty lines, and sort the list which
+ *  assist with matching the static cache
  * @param {string} source requirements
  * @param {string} target requirements where results are written
  * @param {Object} options
@@ -163,8 +185,13 @@ function generateRequirementsFile(source, target, options) {
     .readFileSync(source, { encoding: 'utf-8' })
     .split(/\r?\n/);
   const filteredRequirements = requirements.filter(req => {
+    req = req.trim();
+    if (req.length == 0 || req[0] == '#') {
+      return false;
+    }
     return !noDeploy.has(req.split(/[=<> \t]/)[0].trim());
   });
+  filteredRequirements.sort();  // Sort them alphabetically
   fse.writeFileSync(target, filteredRequirements.join('\n'));
 }
 
@@ -177,15 +204,15 @@ function generateRequirementsFile(source, target, options) {
  */
 function copyVendors(vendorFolder, targetFolder, serverless) {
   // Create target folder if it does not exist
-  const targetRequirementsFolder = path.join(targetFolder, 'requirements');
+  fse.ensureDirSync(targetFolder);
 
   serverless.cli.log(
     `Copying vendor libraries from ${vendorFolder} to ${targetRequirementsFolder}...`
   );
 
   fse.readdirSync(vendorFolder).map(file => {
     let source = path.join(vendorFolder, file);
-    let dest = path.join(targetRequirementsFolder, file);
+    let dest = path.join(targetFolder, file);
     if (fse.existsSync(dest)) {
       rimraf.sync(dest);
     }
@@ -194,11 +221,103 @@ function copyVendors(vendorFolder, targetFolder, serverless) {
 }
 
 /**
- * pip install the requirements to the .serverless/requirements directory
+ * This evaluates if requirements are actually needed to be installed, but fails
+ * gracefully if no req file is found intentionally.  It also assists with code
+ * re-use for this logic pertaining to individually packaged functions
+ * @param {string} servicePath
+ * @param {string} modulePath
+ * @param {Object} options
+ * @param {Object} serverless
+ * @return {string}
+ */
+function installRequirementsIfNeeded(servicePath, modulePath, options, serverless) {
+  // Our source requirements, under our service path, and our module path (if specified)
+  const fileName = path.join(servicePath, modulePath, options.fileName);
+
+  // First, generate the requirements file to our local .serverless folder
+  fse.ensureDirSync(path.join(servicePath, '.serverless'));
+  const slsReqsTxt = path.join(servicePath, '.serverless', 'requirements.txt');
+
+  installRequirementsFile(
+    fileName,
+    slsReqsTxt,
+    serverless,
+    servicePath,
+    options
+  );
+
+  // If no requirements file or an empty requirements file, then do nothing
+  if (!fse.existsSync(slsReqsTxt) || fse.statSync(slsReqsTxt).size == 0) {
+    serverless.cli.log(`Skipping empty output requirements.txt file from ${slsReqsTxt}`);
+    return false;
+  }
+
+  // Copy our requirements to another filename in .serverless (incase of individually packaged)
+  if (modulePath) {
+    destinationFile = path.join(servicePath, '.serverless', modulePath + '_requirements.txt');
+    serverless.cli.log(`Copying from ${slsReqsTxt} into ${destinationFile} ...`);
+    fse.copySync(slsReqsTxt, destinationFile);
+  }
+
+  // Then generate our MD5 Sum of this requirements file to determine where it should "go" to and/or pull cache from
+  const reqChecksum = md5Path(slsReqsTxt);
+
+  // Then figure out where this cache should be, if we're caching, etc
+  const workingReqsFolder = getRequirementsWorkingPath(reqChecksum, servicePath, options);
+
+  // Check if our static cache is present and is valid
+  if (fse.existsSync(workingReqsFolder)) {
+    if (fse.existsSync(path.join(workingReqsFolder, '.completed_requirements'))) {
+      serverless.cli.log(`Using static cache of requirements found at ${workingReqsFolder} ...`);
+      // We'll "touch" the folder, as to bring it to the start of the FIFO cache
+      fse.utimesSync(workingReqsFolder, new Date(), new Date());
+      return workingReqsFolder;
+    }
+    // Remove our old folder if it didn't complete properly, but _just incase_ only remove it if named properly...
+    if (workingReqsFolder.endsWith('_slspyc') || workingReqsFolder.endsWith('.requirements')) {
+      rimraf.sync(workingReqsFolder);
+    }
+  }
+
+  // Ensuring the working reqs folder exists
+  fse.ensureDirSync(workingReqsFolder);
+
+  // Copy our requirements.txt into our working folder...
+  fse.copySync(slsReqsTxt, path.join(workingReqsFolder, 'requirements.txt'));
+
+  // Then install our requirements from this folder
+  installRequirements(
+    workingReqsFolder,
+    serverless,
+    options
+  );
+  if (options.vendor) {
+    // copy vendor libraries to requirements folder
+    copyVendors(
+      options.vendor,
+      workingReqsFolder,
+      serverless
+    );
+  }
+
+  // Then touch our ".completed_requirements" file so we know we can use this for static cache
+  if (options.useStaticCache) {
+    fse.closeSync(fse.openSync(path.join(workingReqsFolder, '.completed_requirements'), 'w'));
+  }
+  return workingReqsFolder;
+}
+
+
+/**
+ * pip install the requirements to the requirements directory
  * @return {undefined}
  */
 function installAllRequirements() {
-  fse.ensureDirSync(path.join(this.servicePath, '.serverless'));
+  // fse.ensureDirSync(path.join(this.servicePath, '.serverless'));
+  // First, check and delete cache versions, if enabled
+  checkForAndDeleteMaxCacheVersions(this.options, this.serverless);
+
+  // Then if we're going to package functions individually...
   if (this.serverless.service.package.individually) {
     let doneModules = [];
     values(this.serverless.service.functions)
@@ -211,36 +330,27 @@ function installAllRequirements() {
         if (!get(f, 'module')) {
           set(f, ['module'], '.');
         }
+        // If we didn't already process a module (functions can re-use modules)
         if (!doneModules.includes(f.module)) {
-          installRequirements(
-            path.join(f.module, this.options.fileName),
-            path.join('.serverless', f.module),
-            this.serverless,
-            this.servicePath,
-            this.options
+          const reqsInstalledAt = installRequirementsIfNeeded(
+            this.servicePath, f.module, this.options, this.serverless
           );
-          if (f.vendor) {
-            // copy vendor libraries to requirements folder
-            copyVendors(
-              f.vendor,
-              path.join('.serverless', f.module),
-              this.serverless
-            );
+          // Add symlinks into .serverless for each module so it's easier for injecting and for users to see where reqs are
+          let symlinkPath = path.join(this.servicePath, '.serverless', join([`${f.module}`,'_requirements'].join('_')));
+          if (reqsInstalledAt && !fse.existsSync(symlinkPath)) {
+            fse.symlink( reqsInstalledAt, symlinkPath );
           }
           doneModules.push(f.module);
         }
       });
   } else {
-    installRequirements(
-      this.options.fileName,
-      '.serverless',
-      this.serverless,
-      this.servicePath,
-      this.options
+    const reqsInstalledAt = installRequirementsIfNeeded(
+      this.servicePath, '', this.options, this.serverless
     );
-    if (this.options.vendor) {
-      // copy vendor libraries to requirements folder
-      copyVendors(this.options.vendor, '.serverless', this.serverless);
+    // Add symlinks into .serverless for so it's easier for injecting and for users to see where reqs are
+    let symlinkPath = path.join(this.servicePath, '.serverless', `requirements`);
+    if (reqsInstalledAt && !fse.existsSync(symlinkPath)) {
+      fse.symlink(reqsInstalledAt, symlinkPath);
     }
   }
 }