Merge pull request #1 from vklab/vklab/per-function-requirements

allow per-function requirements plus base layer

Author: wmarshall

lib/inject.js

@@ -6,6 +6,7 @@ const set = require('lodash.set');
 const path = require('path');
 const JSZip = require('jszip');
 const { writeZip, zipFile } = require('./zipTree');
+const { sha256Hash } = require('./shared')
 
 BbPromise.promisifyAll(fse);
 
@@ -16,9 +17,13 @@ BbPromise.promisifyAll(fse);
  * @param {Object} options our options object
  * @return {Promise} the JSZip object constructed.
  */
-function injectRequirements(requirementsPath, packagePath, options) {
+function injectRequirements(requirementsPath, packagePath, options, outputPackagePath) {
   const noDeploy = new Set(options.noDeploy || []);
 
+  if (!outputPackagePath){
+    outputPackagePath = packagePath;
+  }
+
   return fse
     .readFileAsync(packagePath)
     .then(buffer => JSZip.loadAsync(buffer))
@@ -45,7 +50,7 @@ function injectRequirements(requirementsPath, packagePath, options) {
             createFolders: false
           })
         )
-        .then(() => writeZip(zip, packagePath))
+        .then(() => writeZip(zip, outputPackagePath))
     );
 }
 
@@ -87,7 +92,7 @@ function moveModuleUp(source, target, module) {
  * @return {Promise} the combined promise for requirements injection.
  */
 function injectAllRequirements(funcArtifact) {
-  if (this.options.layer) {
+  if (this.options.layer && !this.serverless.service.package.individually) {
     // The requirements will be placed in a Layer, so just resolve
     return BbPromise.resolve();
   }
@@ -118,18 +123,27 @@ function injectAllRequirements(funcArtifact) {
           return moveModuleUp(artifact, newArtifact, func.module).then(
             () => func
           );
-        } else {
+        }  else {
           return func;
         }
       })
       .map(func => {
-        return this.options.zip
-          ? func
-          : injectRequirements(
-              path.join('.serverless', func.module, 'requirements'),
-              func.package.artifact,
-              this.options
-            );
+        if (this.options.zip) {
+          return func
+        } else if (get(func, 'pythonRequirements.fileName')){
+          const extraRequirementsPath = get(func, 'pythonRequirements.fileName');
+          const extraRequirementsPathHash = sha256Hash(extraRequirementsPath);
+          return injectRequirements(
+            path.join('.serverless', extraRequirementsPathHash, 'requirements'),
+            func.package.artifact,
+            this.options
+          );
+        }
+        return injectRequirements(
+            path.join('.serverless', func.module, 'requirements'),
+            func.package.artifact,
+            this.options
+          );
       });
   } else if (!this.options.zip) {
     return injectRequirements(

lib/pip.js

@@ -11,9 +11,11 @@ const {
   checkForAndDeleteMaxCacheVersions,
   sha256Path,
   getRequirementsWorkingPath,
-  getUserCachePath
+  getUserCachePath,
+  sha256Hash,
 } = require('./shared');
 
+
 /**
  * Omit empty commands.
  * In this context, a "command" is a list of arguments. An empty list or falsy value is ommitted.
@@ -469,23 +471,28 @@ function installRequirementsIfNeeded(
   modulePath,
   options,
   funcOptions,
-  serverless
+  serverless,
+  extraTargetPath
 ) {
   // Our source requirements, under our service path, and our module path (if specified)
   const fileName = path.join(servicePath, modulePath, options.fileName);
+  if (!extraTargetPath) {
+    extraTargetPath = modulePath;
+  }
 
   // Skip requirements generation, if requirements file doesn't exist
   if (!requirementsFileExists(servicePath, options, fileName)) {
+    serverless.cli.log(`Could not find requirements file at ${fileName}`)
     return false;
   }
 
   let requirementsTxtDirectory;
   // Copy our requirements to another path in .serverless (incase of individually packaged)
-  if (modulePath && modulePath !== '.') {
+  if (extraTargetPath && extraTargetPath !== '.') {
     requirementsTxtDirectory = path.join(
       servicePath,
       '.serverless',
-      modulePath
+      extraTargetPath
     );
   } else {
     requirementsTxtDirectory = path.join(servicePath, '.serverless');
@@ -579,6 +586,7 @@ function installAllRequirements() {
   // Then if we're going to package functions individually...
   if (this.serverless.service.package.individually) {
     let doneModules = [];
+    let doneExtraRequirements = [];
     this.targetFuncs
       .filter(func =>
         (func.runtime || this.serverless.service.provider.runtime).match(
@@ -617,14 +625,61 @@ function installAllRequirements() {
               if (process.platform == 'win32') {
                 fse.copySync(reqsInstalledAt, modulePath);
               } else {
-                fse.symlink(reqsInstalledAt, modulePath);
+                fse.symlinkSync(reqsInstalledAt, modulePath);
               }
             } else {
-              fse.rename(reqsInstalledAt, modulePath);
+              fse.renameSync(reqsInstalledAt, modulePath);
             }
           }
           doneModules.push(f.module);
         }
+        return f;
+      })
+      .map(f => {
+        const extraRequirementsPath = get(f, 'pythonRequirements.fileName');
+        this.serverless.cli.log(`Found extra requirements ${extraRequirementsPath} for ${f.name}`)
+        if (extraRequirementsPath) {
+          const extraRequirementsPathHash = sha256Hash(extraRequirementsPath);
+          // this.serverless.cli.log(`Computed path hash ${extraRequirementsPathHash}, doneExtraRequirements=${doneExtraRequirements}`)
+          // If we didn't already process a module (functions can re-use modules)
+          if (!doneExtraRequirements.includes(extraRequirementsPath)) {
+            const reqsInstalledAt = installRequirementsIfNeeded(
+              this.servicePath,
+              '',
+              Object.assign({}, this.options, f.pythonRequirements),
+              f,
+              this.serverless,
+              extraRequirementsPathHash
+            );
+            // this.serverless.cli.log(`Installed reqs to ${reqsInstalledAt}`)
+            // Add modulePath into .serverless for each module so it's easier for injecting and for users to see where reqs are
+            let extraRequirementsDirectoryPath = path.join(
+              this.servicePath,
+              '.serverless',
+              extraRequirementsPathHash,
+              'requirements'
+            );
+            // Only do if we didn't already do it
+            if (
+              reqsInstalledAt &&
+              !fse.existsSync(extraRequirementsDirectoryPath) &&
+              reqsInstalledAt != extraRequirementsDirectoryPath
+            ) {
+              if (this.options.useStaticCache) {
+                // Windows can't symlink so we have to copy on Windows,
+                // it's not as fast, but at least it works
+                if (process.platform == 'win32') {
+                  fse.copySync(reqsInstalledAt, extraRequirementsDirectoryPath);
+                } else {
+                  fse.symlinkSync(reqsInstalledAt, extraRequirementsDirectoryPath);
+                }
+              } else {
+                fse.renameSync(reqsInstalledAt, extraRequirementsDirectoryPath);
+              }
+            }
+            doneExtraRequirements.push(extraRequirementsPath);
+          }
+        }
       });
   } else {
     const reqsInstalledAt = installRequirementsIfNeeded(

lib/shared.js

@@ -4,6 +4,12 @@ const glob = require('glob-all');
 const path = require('path');
 const fse = require('fs-extra');
 const sha256File = require('sha256-file');
+const { createHash } = require('crypto')
+const sha256Hash = (s) => {
+  const hash = createHash('sha256');
+  hash.update(s);
+  return hash.digest('hex');
+}
 
 /**
  * This helper will check if we're using static cache and have max
@@ -108,5 +114,6 @@ module.exports = {
   checkForAndDeleteMaxCacheVersions,
   getRequirementsWorkingPath,
   getUserCachePath,
-  sha256Path
+  sha256Path,
+  sha256Hash
 };

lib/zip.js

@@ -6,6 +6,7 @@ const uniqBy = require('lodash.uniqby');
 const BbPromise = require('bluebird');
 const JSZip = require('jszip');
 const { addTree, writeZip } = require('./zipTree');
+const { sha256Hash } = require('./shared')
 
 BbPromise.promisifyAll(fse);
 
@@ -101,16 +102,22 @@ function packRequirements() {
           }
           return f;
         })
-        .then(funcs => uniqBy(funcs, f => f.module))
+        .then(funcs => uniqBy(funcs, f => get(f, 'pythonRequirements.fileName', f.module)))
         .map(f => {
+          const extraRequirementsPath = get(f, 'pythonRequirements.fileName');
+          let extraRequirementsPathHash = f.module;
+          if (extraRequirementsPath){
+            extraRequirementsPathHash = sha256Hash(extraRequirementsPath);
+          }
+
           this.serverless.cli.log(
-            `Zipping required Python packages for ${f.module}...`
+            `Zipping required Python packages for ${extraRequirementsPathHash}...`
           );
-          f.package.include.push(`${f.module}/.requirements.zip`);
+          f.package.include.push(`${extraRequirementsPathHash}/.requirements.zip`);
           return addTree(
             new JSZip(),
-            `.serverless/${f.module}/requirements`
-          ).then(zip => writeZip(zip, `${f.module}/.requirements.zip`));
+            `.serverless/${extraRequirementsPathHash}/requirements`
+          ).then(zip => writeZip(zip, `${extraRequirementsPathHash}/.requirements.zip`));
         });
     } else {
       this.serverless.cli.log('Zipping required Python packages...');