Static and download caching, rebased and Windows working

Author: Farley

.gitignore

@@ -42,3 +42,6 @@ admin.env
 #PYTHON STUFF
 *.py[co]
 __pycache__
+
+#NODE STUFF
+package-lock.json

README.md

@@ -140,25 +140,40 @@ custom:
 ```
 
 ## Extra Config Options
-### extra pip arguments
-You can specify extra arguments to be passed to pip like this:
+### Caching
+You can enable two kinds of caching with this plugin which are currently both DISABLED by default.  First, a download cache that will cache downloads that pip needs to compile the packages.  And second, a what we call "static caching" which caches output of pip after compiling everything for your requirements file.  Since generally requirements.txt files rarely change, you will often see large amounts of speed improvements when enabling the static cache feature.  These caches will be shared between all your projects if no custom cacheLocation is specified (see below).
+
+ _**Please note:** This has replaced the previously recommended usage of "--cache-dir" in the pipCmdExtraArgs_
 ```yaml
 custom:
   pythonRequirements:
-      dockerizePip: true
-      pipCmdExtraArgs:
-          - --cache-dir
-          - .requirements-cache
+    useDownloadCache: true
+      useStaticCache: true
 ```
+_Additionally, In future versions of this plugin, both caching features will probably be enabled by default_
 
-When using `--cache-dir` don't forget to also exclude it from the package.
+### Other caching options...
+There are two additional options related to caching.  You can specify where in your system that this plugin caches with the `cacheLocation` option.  By default it will figure out automatically where based on your username and your OS to store the cache via the [appdirectory](https://www.npmjs.com/package/appdirectory) module.  Additionally, you can specify how many max static caches to store with `staticCacheMaxVersions`, as a simple attempt to limit disk space usage for caching.  This is DISABLED (set to 0) by default.  Example:
+```yaml
+custom:
+  pythonRequirements:
+      useStaticCache: true
+      useDownloadCache: true
+      cacheLocation: '/home/user/.my_cache_goes_here'
+      staticCacheMaxVersions: 10
+      
+```
 
+### Extra pip arguments
+You can specify extra arguments [supported by pip](https://pip.pypa.io/en/stable/reference/pip_install/#options) to be passed to pip like this:
 ```yaml
-package:
-  exclude:
-    - .requirements-cache/**
+custom:
+  pythonRequirements:
+      pipCmdExtraArgs:
+          - --compile
 ```
 
+
 ### Customize requirements file name
 [Some `pip` workflows involve using requirements files not named
 `requirements.txt`](https://www.kennethreitz.org/essays/a-better-pip-workflow).
@@ -350,4 +365,4 @@ zipinfo .serverless/xxx.zip
    improved pip chache support when using docker.
  * [@dee-me-tree-or-love](https://github.com/dee-me-tree-or-love) - the `slim` package option
  * [@alexjurkiewicz](https://github.com/alexjurkiewicz) - [docs about docker workflows](#native-code-dependencies-during-build)
-
+ * [@andrewfarley](https://github.com/andrewfarley) - Implemented download caching and static caching

index.js

@@ -12,7 +12,7 @@ const {
 const { injectAllRequirements } = require('./lib/inject');
 const { installAllRequirements } = require('./lib/pip');
 const { pipfileToRequirements } = require('./lib/pipenv');
-const { cleanup } = require('./lib/clean');
+const { cleanup, cleanupCache } = require('./lib/clean');
 
 BbPromise.promisifyAll(fse);
 
@@ -39,6 +39,10 @@ class ServerlessPythonRequirements {
         dockerSsh: false,
         dockerImage: null,
         dockerFile: null,
+        useStaticCache: false,
+        useDownloadCache: false,
+        cacheLocation: false,
+        staticCacheMaxVersions: 0,
         pipCmdExtraArgs: [],
         noDeploy: [
           'boto3',
@@ -115,6 +119,11 @@ class ServerlessPythonRequirements {
           install: {
             usage: 'install requirements manually',
             lifecycleEvents: ['install']
+          },
+          cleanCache: {
+            usage:
+              'Removes all items in the pip download/static cache (if present)',
+            lifecycleEvents: ['cleanCache']
           }
         }
       }
@@ -128,6 +137,11 @@ class ServerlessPythonRequirements {
       return args[1].functionObj.runtime.startsWith('python');
     };
 
+    const clean = () =>
+      BbPromise.bind(this)
+        .then(cleanup)
+        .then(removeVendorHelper);
+
     const before = () => {
       if (!isFunctionRuntimePython(arguments)) {
         return;
@@ -155,13 +169,13 @@ class ServerlessPythonRequirements {
 
     const invalidateCaches = () => {
       if (this.options.invalidateCaches) {
-        return BbPromise.bind(this)
-          .then(cleanup)
-          .then(removeVendorHelper);
+        return clean;
       }
       return BbPromise.resolve();
     };
 
+    const cleanCache = () => BbPromise.bind(this).then(cleanupCache);
+
     this.hooks = {
       'after:package:cleanup': invalidateCaches,
       'before:package:createDeploymentArtifacts': before,
@@ -172,16 +186,9 @@ class ServerlessPythonRequirements {
         this.serverless.cli.generateCommandsHelp(['requirements']);
         return BbPromise.resolve();
       },
-      'requirements:install:install': () =>
-        BbPromise.bind(this)
-          .then(pipfileToRequirements)
-          .then(addVendorHelper)
-          .then(installAllRequirements)
-          .then(packRequirements),
-      'requirements:clean:clean': () =>
-        BbPromise.bind(this)
-          .then(cleanup)
-          .then(removeVendorHelper)
+      'requirements:install:install': before,
+      'requirements:clean:clean': clean,
+      'requirements:cleanCache:cleanCache': cleanCache
     };
   }
 }

lib/clean.js

@@ -1,6 +1,8 @@
 const BbPromise = require('bluebird');
 const fse = require('fs-extra');
 const path = require('path');
+const glob = require('glob-all');
+const { getUserCachePath } = require('./shared');
 
 BbPromise.promisifyAll(fse);
 
@@ -29,4 +31,32 @@ function cleanup() {
   );
 }
 
-module.exports = { cleanup };
+/**
+ * Clean up static cache, remove all items in there
+ * @return {Promise}
+ */
+function cleanupCache() {
+  const cacheLocation = getUserCachePath(this.options);
+  if (fse.existsSync(cacheLocation)) {
+    if (this.serverless) {
+      this.serverless.cli.log(`Removing static caches at: ${cacheLocation}`);
+    }
+
+    // Only remove cache folders that we added, just incase someone accidentally puts a weird
+    // static cache location so we don't remove a bunch of personal stuff
+    const promises = [];
+    glob
+      .sync([path.join(cacheLocation, '*slspyc/')], { mark: true, dot: false })
+      .forEach(file => {
+        promises.push(fse.removeAsync(file));
+      });
+    return BbPromise.all(promises);
+  } else {
+    if (this.serverless) {
+      this.serverless.cli.log(`No static cache found`);
+    }
+    return BbPromise.resolve();
+  }
+}
+
+module.exports = { cleanup, cleanupCache };

lib/docker.js

@@ -49,8 +49,11 @@ function findTestFile(servicePath) {
   if (fse.pathExistsSync(path.join(servicePath, 'serverless.json'))) {
     return 'serverless.json';
   }
+  if (fse.pathExistsSync(path.join(servicePath, 'requirements.txt'))) {
+    return 'requirements.txt';
+  }
   throw new Error(
-    'Unable to find serverless.yml or serverless.yaml or serverless.json for getBindPath()'
+    'Unable to find serverless.{yml|yaml|json} or requirements.txt for getBindPath()'
   );
 }
 
@@ -154,7 +157,7 @@ function getDockerUid(bindPath) {
     'stat',
     '-c',
     '%u',
-    '/test/.serverless'
+    '/bin/sh'
   ];
   const ps = dockerCommand(options);
   return ps.stdout.trim();