Reduce size of installed dependencies by stripping (serverless#191)

Author: dee-me-tree-or-love

README.md

@@ -95,7 +95,28 @@ try:
 except ImportError:
   pass
 ```
-
+### Slim Package
+_Works on non 'win32' environments: Docker, WSL are included_  
+To remove the tests, information and caches from the installed packages, 
+enable the `slim` option. This will: `strip` the `.so` files, remove `__pycache__` 
+directories and `dist-info` directories.  
+```yaml
+custom:
+  pythonRequirements:
+    slim: true
+```  
+#### Custom Removal Patterns  
+To specify additional directories to remove from the installed packages, 
+define the patterns using regex as a `slimPatterns` option in serverless config:  
+```yaml
+custom:
+  pythonRequirements:
+    slim: true
+    slimPatterns:
+      - "*.egg-info*"
+```  
+This will remove all folders within the installed requirements that match 
+the names in `slimPatterns`  
 ## Omitting Packages 
 You can omit a package from deployment with the `noDeploy` option. Note that
 dependencies of omitted packages must explicitly be omitted too.
@@ -266,3 +287,5 @@ For usage of `dockerizePip` on Windows do Step 1 only if running serverless on w
  * [@kichik](https://github.com/kichik) - Imposed windows & `noDeploy` support,
    switched to adding files straight to zip instead of creating symlinks, and
    improved pip chache support when using docker.
+ * [@dee-me-tree-or-love](https://github.com/dee-me-tree-or-love) - the `slim` package option
+

index.js

@@ -26,6 +26,7 @@ class ServerlessPythonRequirements {
   get options() {
     const options = Object.assign(
       {
+        slim: false,
         zip: false,
         cleanupZipHelper: true,
         invalidateCaches: false,

lib/pip.js

@@ -6,6 +6,7 @@ const set = require('lodash.set');
 const { spawnSync } = require('child_process');
 const values = require('lodash.values');
 const { buildImage, getBindPath, getDockerUid } = require('./docker');
+const { getSlimPackageCommands } = require('./slim');
 
 /**
  * Install requirements described in requirementsPath to targetFolder
@@ -122,6 +123,14 @@ function installRequirements(
     cmd = pipCmd[0];
     cmdOptions = pipCmd.slice(1);
   }
+
+  // If enabled slimming, strip out the caches, tests and dist-infos
+  if (options.slim === true || options.slim === 'true') {
+    const preparedPath = dockerPathForWin(options, targetRequirementsFolder);
+    const slimCmd = getSlimPackageCommands(options, preparedPath);
+    cmdOptions.push(...slimCmd);
+  }
+
   const res = spawnSync(cmd, cmdOptions, { cwd: servicePath, shell: true });
   if (res.error) {
     if (res.error.code === 'ENOENT') {

lib/slim.js

@@ -0,0 +1,59 @@
+const isWsl = require('is-wsl');
+
+/**
+ * Get commands to slim the installed requirements
+ * only for non-windows platforms:
+ * works for docker builds and when run on UNIX platforms (wsl included)
+ * @param {Object} options
+ * @param {string} folderPath
+ * @return {Array.<String>}
+ */
+function getSlimPackageCommands(options, folderPath) {
+  let stripCmd = [];
+
+  // Default stripping is done for non-windows environments
+  if (process.platform !== 'win32' || isWsl) {
+    stripCmd = getDefaultSLimOptions(folderPath);
+
+    // If specified any custom patterns to remove
+    if (options.slimPatterns instanceof Array) {
+      // Add the custom specified patterns to remove to the default commands
+      const customPatterns = options.slimPatterns.map(pattern => {
+        return getRemovalCommand(folderPath, pattern);
+      });
+      stripCmd = stripCmd.concat(customPatterns);
+    }
+  }
+  return stripCmd;
+}
+
+/**
+ * Gets the commands to slim the default (safe) files:
+ * including removing caches, stripping compiled files, removing dist-infos
+ * @param {String} folderPath
+ * @return {Array}
+ */
+function getDefaultSLimOptions(folderPath) {
+  return [
+    `&& find ${folderPath} -name "*.so" -exec strip {} \\;`,
+    `&& find ${folderPath} -name "*.py[c|o]" -exec rm -rf {} +`,
+    `&& find ${folderPath} -type d -name "__pycache__*" -exec rm -rf {} +`,
+    `&& find ${folderPath} -type d -name "*.dist-info*" -exec rm -rf {} +`
+  ];
+}
+
+/**
+ * Get the command created fromt he find and remove template:
+ * returns a string in form `&& find <folder> -name "<match>" -exec rm -rf {} +`
+ * @param {String} folderPath
+ * @param {String} removalMatch
+ * @return {String}
+ */
+function getRemovalCommand(folderPath, removalMatch) {
+  return `&& find ${folderPath} -type d -name "${removalMatch}" -exec rm -rf {} +`;
+}
+
+module.exports = {
+  getSlimPackageCommands,
+  getDefaultSLimOptions
+};

test.bats

@@ -32,6 +32,27 @@ teardown() {
     ! ls puck/flask
 }
 
+@test "py3.6 can package flask with slim options" {
+    cd tests/base
+    npm i $(npm pack ../..)
+    sls --slim=true package
+    unzip .serverless/sls-py-req-test.zip -d puck
+    ls puck/flask
+    test $(find puck -name "*.pyc" | wc -l) -eq 0
+}
+
+@test "py3.6 can package flask with slim & slimPatterns options" {
+    cd tests/base
+    mv _slimPatterns.yml slimPatterns.yml
+    npm i $(npm pack ../..)
+    sls --slim=true package
+    mv slimPatterns.yml _slimPatterns.yml    
+    unzip .serverless/sls-py-req-test.zip -d puck
+    ls puck/flask
+    test $(find puck -name "*.pyc" | wc -l) -eq 0
+    test $(find puck -type d -name "*.egg-info*" | wc -l) -eq 0  
+}
+
 @test "py3.6 doesn't package boto3 by default" {
     cd tests/base
     npm i $(npm pack ../..)
@@ -59,6 +80,15 @@ teardown() {
     ls puck/.requirements.zip puck/unzip_requirements.py
 }
 
+@test "py3.6 can package flask with zip & slim & dockerizePip option" {
+    cd tests/base
+    npm i $(npm pack ../..)
+    ! uname -sm|grep Linux || groups|grep docker || id -u|egrep '^0$' || skip "can't dockerize on linux if not root & not in docker group"
+    sls --dockerizePip=true --zip=true --slim=true package
+    unzip .serverless/sls-py-req-test.zip -d puck
+    ls puck/.requirements.zip puck/unzip_requirements.py
+}
+
 @test "py3.6 can package flask with dockerizePip option" {
     cd tests/base
     npm i $(npm pack ../..)
@@ -68,6 +98,29 @@ teardown() {
     ls puck/flask
 }
 
+@test "py3.6 can package flask with slim & dockerizePip option" {
+    cd tests/base
+    npm i $(npm pack ../..)
+    ! uname -sm|grep Linux || groups|grep docker || id -u|egrep '^0$' || skip "can't dockerize on linux if not root & not in docker group"
+    sls --dockerizePip=true --slim=true package
+    unzip .serverless/sls-py-req-test.zip -d puck
+    ls puck/flask
+    test $(find puck -name "*.pyc" | wc -l) -eq 0
+}
+
+@test "py3.6 can package flask with slim & dockerizePip & slimPatterns options" {
+    cd tests/base
+    mv _slimPatterns.yml slimPatterns.yml
+    npm i $(npm pack ../..)
+    ! uname -sm|grep Linux || groups|grep docker || id -u|egrep '^0$' || skip "can't dockerize on linux if not root & not in docker group"
+    sls --dockerizePip=true --slim=true package
+    mv slimPatterns.yml _slimPatterns.yml    
+    unzip .serverless/sls-py-req-test.zip -d puck
+    ls puck/flask
+    test $(find puck -name "*.pyc" | wc -l) -eq 0
+    test $(find puck -type d -name "*.egg-info*" | wc -l) -eq 0  
+}
+
 @test "py3.6 uses cache with dockerizePip option" {
     cd tests/base
     npm i $(npm pack ../..)
@@ -77,6 +130,17 @@ teardown() {
     ls .requirements-cache/http
 }
 
+@test "py3.6 uses cache with dockerizePip & slim option" {
+    cd tests/base
+    npm i $(npm pack ../..)
+    ! uname -sm|grep Linux || groups|grep docker || id -u|egrep '^0$' || skip "can't dockerize on linux if not root & not in docker group"
+    perl -p -i'.bak' -e 's/(pythonRequirements:$)/\1\n    pipCmdExtraArgs: ["--cache-dir", ".requirements-cache"]/' serverless.yml
+    sls --dockerizePip=true --slim=true package
+    ls .requirements-cache/http
+    test $(find puck -name "*.pyc" | wc -l) -eq 0
+}
+
+
 @test "py2.7 can package flask with default options" {
     cd tests/base
     npm i $(npm pack ../..)
@@ -85,6 +149,15 @@ teardown() {
     ls puck/flask
 }
 
+@test "py2.7 can package flask with slim option" {
+    cd tests/base
+    npm i $(npm pack ../..)
+    sls --runtime=python2.7 --slim=true package 
+    unzip .serverless/sls-py-req-test.zip -d puck
+    ls puck/flask
+    test $(find puck -name "*.pyc" | wc -l) -eq 0
+}
+
 @test "py2.7 can package flask with zip option" {
     cd tests/base
     npm i $(npm pack ../..)
@@ -93,6 +166,18 @@ teardown() {
     ls puck/.requirements.zip puck/unzip_requirements.py
 }
 
+@test "py2.7 can package flask with slim & dockerizePip & slimPatterns options" {
+    cd tests/base
+    mv _slimPatterns.yml slimPatterns.yml
+    npm i $(npm pack ../..)
+    sls --runtime=python2.7 --slim=true packag
+    mv slimPatterns.yml _slimPatterns.yml    
+    unzip .serverless/sls-py-req-test.zip -d puck
+    ls puck/flask
+    test $(find puck -name "*.pyc" | wc -l) -eq 0
+    test $(find puck -type d -name "*.egg-info*" | wc -l) -eq 0  
+}
+
 @test "py2.7 doesn't package boto3 by default" {
     cd tests/base
     npm i $(npm pack ../..)
@@ -119,6 +204,15 @@ teardown() {
     ls puck/.requirements.zip puck/unzip_requirements.py
 }
 
+@test "py2.7 can package flask with zip & slim & dockerizePip option" {
+    cd tests/base
+    npm i $(npm pack ../..)
+    ! uname -sm|grep Linux || groups|grep docker || id -u|egrep '^0$' || skip "can't dockerize on linux if not root & not in docker group"
+    sls --dockerizePip=true --runtime=python2.7 --zip=true --slim=true package
+    unzip .serverless/sls-py-req-test.zip -d puck
+    ls puck/.requirements.zip puck/unzip_requirements.py
+}
+
 @test "py2.7 can package flask with dockerizePip option" {
     cd tests/base
     npm i $(npm pack ../..)
@@ -128,6 +222,29 @@ teardown() {
     ls puck/flask
 }
 
+@test "py2.7 can package flask with slim & dockerizePip option" {
+    cd tests/base
+    npm i $(npm pack ../..)
+    ! uname -sm|grep Linux || groups|grep docker || id -u|egrep '^0$' || skip "can't dockerize on linux if not root & not in docker group"
+    sls --dockerizePip=true --slim=true --runtime=python2.7 package
+    unzip .serverless/sls-py-req-test.zip -d puck
+    ls puck/flask
+    test $(find puck -name "*.pyc" | wc -l) -eq 0
+}
+
+@test "py2.7 can package flask with slim & dockerizePip & slimPatterns options" {
+    cd tests/base
+    mv _slimPatterns.yml slimPatterns.yml
+    npm i $(npm pack ../..)
+    ! uname -sm|grep Linux || groups|grep docker || id -u|egrep '^0$' || skip "can't dockerize on linux if not root & not in docker group"
+    sls --dockerizePip=true --slim=true --runtime=python2.7 package
+    mv slimPatterns.yml _slimPatterns.yml    
+    unzip .serverless/sls-py-req-test.zip -d puck
+    ls puck/flask
+    test $(find puck -name "*.pyc" | wc -l) -eq 0
+    test $(find puck -type d -name "*.egg-info*" | wc -l) -eq 0  
+}
+
 @test "pipenv py3.6 can package flask with default options" {
     cd tests/pipenv
     npm i $(npm pack ../..)
@@ -136,6 +253,27 @@ teardown() {
     ls puck/flask
 }
 
+@test "pipenv py3.6 can package flask with slim option" {
+    cd tests/pipenv
+    npm i $(npm pack ../..)
+    sls --slim=true package
+    unzip .serverless/sls-py-req-test.zip -d puck
+    ls puck/flask
+    test $(find puck -name "*.pyc" | wc -l) -eq 0
+}
+
+@test "pipenv py3.6 can package flask with slim & slimPatterns option" {
+    cd tests/pipenv
+    npm i $(npm pack ../..)
+    mv _slimPatterns.yml slimPatterns.yml
+    sls --slim=true package
+    mv slimPatterns.yml _slimPatterns.yml    
+    unzip .serverless/sls-py-req-test.zip -d puck
+    ls puck/flask
+    test $(find puck -name "*.pyc" | wc -l) -eq 0
+    test $(find puck -type d -name "*.egg-info*" | wc -l) -eq 0  
+}
+
 @test "pipenv py3.6 can package flask with zip option" {
     cd tests/pipenv
     npm i $(npm pack ../..)
@@ -182,6 +320,20 @@ teardown() {
     ! ls puck3/flask
 }
 
+@test "py3.6 can package flask with package individually & slim option" {
+    cd tests/base
+    npm i $(npm pack ../..)
+    sls --individually=true --slim=true package
+    unzip .serverless/hello.zip -d puck
+    unzip .serverless/hello2.zip -d puck2
+    unzip .serverless/hello3.zip -d puck3
+    ls puck/flask
+    ls puck2/flask
+    ! ls puck3/flask
+    test $(find "puck*" -name "*.pyc" | wc -l) -eq 0
+}
+
+
 @test "py2.7 can package flask with package individually option" {
     cd tests/base
     npm i $(npm pack ../..)
@@ -194,6 +346,20 @@ teardown() {
     ! ls puck3/flask
 }
 
+@test "py2.7 can package flask with package individually & slim option" {
+    cd tests/base
+    npm i $(npm pack ../..)
+    sls --individually=true --slim=true --runtime=python2.7 package
+    unzip .serverless/hello.zip -d puck
+    unzip .serverless/hello2.zip -d puck2
+    unzip .serverless/hello3.zip -d puck3
+    ls puck/flask
+    ls puck2/flask
+    ! ls puck3/flask
+    test $(find puck* -name "*.pyc" | wc -l) -eq 0
+}
+
+
 @test "py3.6 can package only requirements of module" {
     cd tests/individually
     npm i $(npm pack ../..)

tests/base/_slimPatterns.yml

@@ -0,0 +1,2 @@
+slimPatterns:
+  - "*.egg-info*"

tests/base/serverless.yml

@@ -10,8 +10,12 @@ custom:
   pythonRequirements:
     zip: ${opt:zip, self:custom.defaults.zip}
     dockerizePip: ${opt:dockerizePip, self:custom.defaults.dockerizePip}
+    slim: ${opt:slim, self:custom.defaults.slim}
+    slimPatterns: ${file(./slimPatterns.yml):slimPatterns, self:custom.defaults.slimPatterns}
     vendor: ${opt:vendor, ''}
   defaults:
+    slim: false
+    slimPatterns: false
     zip: false
     dockerizePip: false
     individually: false

tests/pipenv/_slimPatterns.yml

@@ -0,0 +1,2 @@
+slimPatterns:
+  - "*.egg-info*"