From c7cd20b79199fb777098f077bef4e7ab67b6bf5b Mon Sep 17 00:00:00 2001 From: Daniel Schep Date: Wed, 8 Aug 2018 15:31:36 -0400 Subject: [PATCH 1/9] basic cleanup aroudn slim option --- lib/pip.js | 6 ------ lib/slim.js | 6 +++--- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/lib/pip.js b/lib/pip.js index f339d364..775d5bb6 100644 --- a/lib/pip.js +++ b/lib/pip.js @@ -106,12 +106,6 @@ function installRequirements( if (process.platform === 'linux') { // Use same user so requirements folder is not root and so --cache-dir works cmdOptions.push('-u', `${process.getuid()}`); - // const stripCmd = quote([ - // 'find', targetRequirementsFolder, - // '-name', '"*.so"', - // '-exec', 'strip', '{}', '\;', - // ]); - // pipCmd = ['/bin/bash', '-c', '"' + pipCmd + ' && ' + stripCmd + ' && ' + chownCmd + '"']; } else { // Use same user so --cache-dir works cmdOptions.push('-u', getDockerUid(bindPath)); diff --git a/lib/slim.js b/lib/slim.js index 2eaf890f..4b4513b9 100644 --- a/lib/slim.js +++ b/lib/slim.js @@ -13,7 +13,7 @@ function getSlimPackageCommands(options, folderPath) { // Default stripping is done for non-windows environments if (process.platform !== 'win32' || isWsl) { - stripCmd = getDefaultSLimOptions(folderPath); + stripCmd = getDefaultSlimOptions(folderPath); // If specified any custom patterns to remove if (options.slimPatterns instanceof Array) { @@ -33,7 +33,7 @@ function getSlimPackageCommands(options, folderPath) { * @param {String} folderPath * @return {Array} */ -function getDefaultSLimOptions(folderPath) { +function getDefaultSlimOptions(folderPath) { return [ `&& find ${folderPath} -name "*.so" -exec strip {} \\;`, `&& find ${folderPath} -name "*.py[c|o]" -exec rm -rf {} +`, @@ -55,5 +55,5 @@ function getRemovalCommand(folderPath, removalMatch) { module.exports = { getSlimPackageCommands, - getDefaultSLimOptions + getDefaultSlimOptions }; From e96ed4c007340a0c936fa899288facd110bff22e Mon Sep 17 00:00:00 2001 From: Daniel Schep Date: Wed, 8 Aug 2018 16:41:16 -0400 Subject: [PATCH 2/9] clean up slim code using glob-all to make it more cross platform --- index.js | 1 + lib/pip.js | 11 +++-- lib/slim.js | 73 ++++++++++------------------------ tests/base/_slimPatterns.yml | 2 +- tests/pipenv/_slimPatterns.yml | 2 +- 5 files changed, 32 insertions(+), 57 deletions(-) diff --git a/index.js b/index.js index 1857b239..2329bed5 100644 --- a/index.js +++ b/index.js @@ -28,6 +28,7 @@ class ServerlessPythonRequirements { const options = Object.assign( { slim: false, + slimPatterns: false, zip: false, cleanupZipHelper: true, invalidateCaches: false, diff --git a/lib/pip.js b/lib/pip.js index 775d5bb6..4a1fc59f 100644 --- a/lib/pip.js +++ b/lib/pip.js @@ -5,7 +5,7 @@ const get = require('lodash.get'); const set = require('lodash.set'); const { spawnSync } = require('child_process'); const { buildImage, getBindPath, getDockerUid } = require('./docker'); -const { getSlimPackageCommands } = require('./slim'); +const { getStripCommand, deleteFiles } = require('./slim'); /** * Install requirements described in requirementsPath to targetFolder @@ -117,11 +117,10 @@ function installRequirements( cmdOptions = pipCmd.slice(1); } - // If enabled slimming, strip out the caches, tests and dist-infos + // If enabled slimming, strip so files if (options.slim === true || options.slim === 'true') { const preparedPath = dockerPathForWin(options, targetRequirementsFolder); - const slimCmd = getSlimPackageCommands(options, preparedPath); - cmdOptions.push(...slimCmd); + cmdOptions.push(getStripCommand(options, preparedPath)); } const res = spawnSync(cmd, cmdOptions, { cwd: servicePath, shell: true }); @@ -139,6 +138,10 @@ function installRequirements( if (res.status !== 0) { throw new Error(res.stderr); } + // If enabled slimming, delete files in slimPatterns + if (options.slim === true || options.slim === 'true') { + deleteFiles(options, targetRequirementsFolder); + } } /** diff --git a/lib/slim.js b/lib/slim.js index 4b4513b9..577072af 100644 --- a/lib/slim.js +++ b/lib/slim.js @@ -1,59 +1,30 @@ const isWsl = require('is-wsl'); +const glob = require('glob-all'); +const fse = require('fs-extra'); -/** - * Get commands to slim the installed requirements - * only for non-windows platforms: - * works for docker builds and when run on UNIX platforms (wsl included) - * @param {Object} options - * @param {string} folderPath - * @return {Array.} - */ -function getSlimPackageCommands(options, folderPath) { - let stripCmd = []; +const getStripCommand = (options, folderPath) => + process.platform !== 'win32' || isWsl || options.dockerizePip + ? ` && find ${folderPath} -name "*.so" -exec strip {} \\;` + : ''; - // Default stripping is done for non-windows environments - if (process.platform !== 'win32' || isWsl) { - stripCmd = getDefaultSlimOptions(folderPath); - - // If specified any custom patterns to remove - if (options.slimPatterns instanceof Array) { - // Add the custom specified patterns to remove to the default commands - const customPatterns = options.slimPatterns.map(pattern => { - return getRemovalCommand(folderPath, pattern); - }); - stripCmd = stripCmd.concat(customPatterns); +const deleteFiles = (options, folderPath) => { + const patterns = [ + '**/*.so', + '**/*.py[c|o]', + '**/__pycache__*', + '**/*.dist-info*' + ]; + if (options.slimPatterns) { + patterns.concat(options.slimPatterns); + } + for (const pattern of patterns) { + for (const file of glob.sync(`${folderPath}/${pattern}`)) { + fse.removeSync(file); } } - return stripCmd; -} - -/** - * Gets the commands to slim the default (safe) files: - * including removing caches, stripping compiled files, removing dist-infos - * @param {String} folderPath - * @return {Array} - */ -function getDefaultSlimOptions(folderPath) { - return [ - `&& find ${folderPath} -name "*.so" -exec strip {} \\;`, - `&& find ${folderPath} -name "*.py[c|o]" -exec rm -rf {} +`, - `&& find ${folderPath} -type d -name "__pycache__*" -exec rm -rf {} +`, - `&& find ${folderPath} -type d -name "*.dist-info*" -exec rm -rf {} +` - ]; -} - -/** - * Get the command created fromt he find and remove template: - * returns a string in form `&& find -name "" -exec rm -rf {} +` - * @param {String} folderPath - * @param {String} removalMatch - * @return {String} - */ -function getRemovalCommand(folderPath, removalMatch) { - return `&& find ${folderPath} -type d -wholename "${removalMatch}" -exec rm -rf {} +`; -} +}; module.exports = { - getSlimPackageCommands, - getDefaultSlimOptions + getStripCommand, + deleteFiles }; diff --git a/tests/base/_slimPatterns.yml b/tests/base/_slimPatterns.yml index ffc3c134..70f863cc 100644 --- a/tests/base/_slimPatterns.yml +++ b/tests/base/_slimPatterns.yml @@ -1,2 +1,2 @@ slimPatterns: - - "*.egg-info*" \ No newline at end of file + - "**/*.egg-info*" diff --git a/tests/pipenv/_slimPatterns.yml b/tests/pipenv/_slimPatterns.yml index ffc3c134..70f863cc 100644 --- a/tests/pipenv/_slimPatterns.yml +++ b/tests/pipenv/_slimPatterns.yml @@ -1,2 +1,2 @@ slimPatterns: - - "*.egg-info*" \ No newline at end of file + - "**/*.egg-info*" From c25a85581c91a4f972b7d6d972a0de5f157c4502 Mon Sep 17 00:00:00 2001 From: Daniel Schep Date: Thu, 9 Aug 2018 09:01:43 -0400 Subject: [PATCH 3/9] fix use of Array.concat --- lib/slim.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/slim.js b/lib/slim.js index 577072af..7357406c 100644 --- a/lib/slim.js +++ b/lib/slim.js @@ -8,14 +8,14 @@ const getStripCommand = (options, folderPath) => : ''; const deleteFiles = (options, folderPath) => { - const patterns = [ + let patterns = [ '**/*.so', '**/*.py[c|o]', '**/__pycache__*', '**/*.dist-info*' ]; if (options.slimPatterns) { - patterns.concat(options.slimPatterns); + patterns = patterns.concat(options.slimPatterns); } for (const pattern of patterns) { for (const file of glob.sync(`${folderPath}/${pattern}`)) { From f3742881f01dbb53e10fee664fbd8efba2846698 Mon Sep 17 00:00:00 2001 From: Evaluating Date: Thu, 23 Aug 2018 11:01:12 -0400 Subject: [PATCH 4/9] try omiting \\ in strip command. might need to differ command for windows --- lib/slim.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/slim.js b/lib/slim.js index 7357406c..d3a0bc69 100644 --- a/lib/slim.js +++ b/lib/slim.js @@ -4,12 +4,11 @@ const fse = require('fs-extra'); const getStripCommand = (options, folderPath) => process.platform !== 'win32' || isWsl || options.dockerizePip - ? ` && find ${folderPath} -name "*.so" -exec strip {} \\;` + ? ` && find ${folderPath} -name "*.so" -exec strip {} ;` : ''; const deleteFiles = (options, folderPath) => { let patterns = [ - '**/*.so', '**/*.py[c|o]', '**/__pycache__*', '**/*.dist-info*' From 7f5d861314f3915c1d44df9a4123b1478f897c90 Mon Sep 17 00:00:00 2001 From: Evaluating Date: Thu, 23 Aug 2018 11:07:28 -0400 Subject: [PATCH 5/9] prettier --- lib/slim.js | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/lib/slim.js b/lib/slim.js index d3a0bc69..79cb7392 100644 --- a/lib/slim.js +++ b/lib/slim.js @@ -8,11 +8,7 @@ const getStripCommand = (options, folderPath) => : ''; const deleteFiles = (options, folderPath) => { - let patterns = [ - '**/*.py[c|o]', - '**/__pycache__*', - '**/*.dist-info*' - ]; + let patterns = ['**/*.py[c|o]', '**/__pycache__*', '**/*.dist-info*']; if (options.slimPatterns) { patterns = patterns.concat(options.slimPatterns); } From 2ef8cae6bde618b0fef4352eb71915815ed974f2 Mon Sep 17 00:00:00 2001 From: Evaluating Date: Thu, 23 Aug 2018 11:54:23 -0400 Subject: [PATCH 6/9] use + with -exec for strip bc strip supports multiple files and + doesnt need to be escaped --- lib/slim.js | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/slim.js b/lib/slim.js index 79cb7392..660bbd7e 100644 --- a/lib/slim.js +++ b/lib/slim.js @@ -4,11 +4,15 @@ const fse = require('fs-extra'); const getStripCommand = (options, folderPath) => process.platform !== 'win32' || isWsl || options.dockerizePip - ? ` && find ${folderPath} -name "*.so" -exec strip {} ;` + ? ` && find ${folderPath} -name "*.so" -exec strip {} +` : ''; const deleteFiles = (options, folderPath) => { - let patterns = ['**/*.py[c|o]', '**/__pycache__*', '**/*.dist-info*']; + let patterns = [ + '**/*.py[c|o]', + '**/__pycache__*', + '**/*.dist-info*' + ]; if (options.slimPatterns) { patterns = patterns.concat(options.slimPatterns); } From 8886f21f1c87296c4ead781079b3a5bc9d61b5c7 Mon Sep 17 00:00:00 2001 From: Evaluating Date: Thu, 23 Aug 2018 12:04:56 -0400 Subject: [PATCH 7/9] doh prettier again --- lib/slim.js | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/lib/slim.js b/lib/slim.js index 660bbd7e..349eac90 100644 --- a/lib/slim.js +++ b/lib/slim.js @@ -8,11 +8,7 @@ const getStripCommand = (options, folderPath) => : ''; const deleteFiles = (options, folderPath) => { - let patterns = [ - '**/*.py[c|o]', - '**/__pycache__*', - '**/*.dist-info*' - ]; + let patterns = ['**/*.py[c|o]', '**/__pycache__*', '**/*.dist-info*']; if (options.slimPatterns) { patterns = patterns.concat(options.slimPatterns); } From b87a3fc06cc7ab426448420da4810138a8736bba Mon Sep 17 00:00:00 2001 From: Evaluating Date: Thu, 23 Aug 2018 14:01:01 -0400 Subject: [PATCH 8/9] damn... find in docker on widnows is finicky --- lib/slim.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/slim.js b/lib/slim.js index 349eac90..ede926ce 100644 --- a/lib/slim.js +++ b/lib/slim.js @@ -4,7 +4,7 @@ const fse = require('fs-extra'); const getStripCommand = (options, folderPath) => process.platform !== 'win32' || isWsl || options.dockerizePip - ? ` && find ${folderPath} -name "*.so" -exec strip {} +` + ? ` && find ${folderPath} -name "*.so" -exec strip {} ';'` : ''; const deleteFiles = (options, folderPath) => { From 442320524b90719cc54af0abf08a6868773331d6 Mon Sep 17 00:00:00 2001 From: Daniel Schep Date: Tue, 28 Aug 2018 08:53:27 -0400 Subject: [PATCH 9/9] Update README.md --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 8c9bf04f..1cec3d6b 100644 --- a/README.md +++ b/README.md @@ -107,13 +107,15 @@ custom: ``` #### Custom Removal Patterns To specify additional directories to remove from the installed packages, -define the patterns using regex as a `slimPatterns` option in serverless config: +define a list of of patterns int he serverless config using the `slimPatterns` +option and glob syntax. Note, it matches against whole paths, so to match a file in any +directory, start your pattern with `**/`. ```yaml custom: pythonRequirements: slim: true slimPatterns: - - "*.egg-info*" + - "**/*.egg-info*" ``` This will remove all folders within the installed requirements that match the names in `slimPatterns`