diff --git a/README.md b/README.md index 8c9bf04f..1cec3d6b 100644 --- a/README.md +++ b/README.md @@ -107,13 +107,15 @@ custom: ``` #### Custom Removal Patterns To specify additional directories to remove from the installed packages, -define the patterns using regex as a `slimPatterns` option in serverless config: +define a list of of patterns int he serverless config using the `slimPatterns` +option and glob syntax. Note, it matches against whole paths, so to match a file in any +directory, start your pattern with `**/`. ```yaml custom: pythonRequirements: slim: true slimPatterns: - - "*.egg-info*" + - "**/*.egg-info*" ``` This will remove all folders within the installed requirements that match the names in `slimPatterns` diff --git a/index.js b/index.js index 1857b239..2329bed5 100644 --- a/index.js +++ b/index.js @@ -28,6 +28,7 @@ class ServerlessPythonRequirements { const options = Object.assign( { slim: false, + slimPatterns: false, zip: false, cleanupZipHelper: true, invalidateCaches: false, diff --git a/lib/pip.js b/lib/pip.js index f339d364..4a1fc59f 100644 --- a/lib/pip.js +++ b/lib/pip.js @@ -5,7 +5,7 @@ const get = require('lodash.get'); const set = require('lodash.set'); const { spawnSync } = require('child_process'); const { buildImage, getBindPath, getDockerUid } = require('./docker'); -const { getSlimPackageCommands } = require('./slim'); +const { getStripCommand, deleteFiles } = require('./slim'); /** * Install requirements described in requirementsPath to targetFolder @@ -106,12 +106,6 @@ function installRequirements( if (process.platform === 'linux') { // Use same user so requirements folder is not root and so --cache-dir works cmdOptions.push('-u', `${process.getuid()}`); - // const stripCmd = quote([ - // 'find', targetRequirementsFolder, - // '-name', '"*.so"', - // '-exec', 'strip', '{}', '\;', - // ]); - // pipCmd = ['/bin/bash', '-c', '"' + pipCmd + ' && ' + stripCmd + ' && ' + chownCmd + '"']; } else { // Use same user so --cache-dir works cmdOptions.push('-u', getDockerUid(bindPath)); @@ -123,11 +117,10 @@ function installRequirements( cmdOptions = pipCmd.slice(1); } - // If enabled slimming, strip out the caches, tests and dist-infos + // If enabled slimming, strip so files if (options.slim === true || options.slim === 'true') { const preparedPath = dockerPathForWin(options, targetRequirementsFolder); - const slimCmd = getSlimPackageCommands(options, preparedPath); - cmdOptions.push(...slimCmd); + cmdOptions.push(getStripCommand(options, preparedPath)); } const res = spawnSync(cmd, cmdOptions, { cwd: servicePath, shell: true }); @@ -145,6 +138,10 @@ function installRequirements( if (res.status !== 0) { throw new Error(res.stderr); } + // If enabled slimming, delete files in slimPatterns + if (options.slim === true || options.slim === 'true') { + deleteFiles(options, targetRequirementsFolder); + } } /** diff --git a/lib/slim.js b/lib/slim.js index 2eaf890f..ede926ce 100644 --- a/lib/slim.js +++ b/lib/slim.js @@ -1,59 +1,25 @@ const isWsl = require('is-wsl'); +const glob = require('glob-all'); +const fse = require('fs-extra'); -/** - * Get commands to slim the installed requirements - * only for non-windows platforms: - * works for docker builds and when run on UNIX platforms (wsl included) - * @param {Object} options - * @param {string} folderPath - * @return {Array.} - */ -function getSlimPackageCommands(options, folderPath) { - let stripCmd = []; +const getStripCommand = (options, folderPath) => + process.platform !== 'win32' || isWsl || options.dockerizePip + ? ` && find ${folderPath} -name "*.so" -exec strip {} ';'` + : ''; - // Default stripping is done for non-windows environments - if (process.platform !== 'win32' || isWsl) { - stripCmd = getDefaultSLimOptions(folderPath); - - // If specified any custom patterns to remove - if (options.slimPatterns instanceof Array) { - // Add the custom specified patterns to remove to the default commands - const customPatterns = options.slimPatterns.map(pattern => { - return getRemovalCommand(folderPath, pattern); - }); - stripCmd = stripCmd.concat(customPatterns); +const deleteFiles = (options, folderPath) => { + let patterns = ['**/*.py[c|o]', '**/__pycache__*', '**/*.dist-info*']; + if (options.slimPatterns) { + patterns = patterns.concat(options.slimPatterns); + } + for (const pattern of patterns) { + for (const file of glob.sync(`${folderPath}/${pattern}`)) { + fse.removeSync(file); } } - return stripCmd; -} - -/** - * Gets the commands to slim the default (safe) files: - * including removing caches, stripping compiled files, removing dist-infos - * @param {String} folderPath - * @return {Array} - */ -function getDefaultSLimOptions(folderPath) { - return [ - `&& find ${folderPath} -name "*.so" -exec strip {} \\;`, - `&& find ${folderPath} -name "*.py[c|o]" -exec rm -rf {} +`, - `&& find ${folderPath} -type d -name "__pycache__*" -exec rm -rf {} +`, - `&& find ${folderPath} -type d -name "*.dist-info*" -exec rm -rf {} +` - ]; -} - -/** - * Get the command created fromt he find and remove template: - * returns a string in form `&& find -name "" -exec rm -rf {} +` - * @param {String} folderPath - * @param {String} removalMatch - * @return {String} - */ -function getRemovalCommand(folderPath, removalMatch) { - return `&& find ${folderPath} -type d -wholename "${removalMatch}" -exec rm -rf {} +`; -} +}; module.exports = { - getSlimPackageCommands, - getDefaultSLimOptions + getStripCommand, + deleteFiles }; diff --git a/tests/base/_slimPatterns.yml b/tests/base/_slimPatterns.yml index ffc3c134..70f863cc 100644 --- a/tests/base/_slimPatterns.yml +++ b/tests/base/_slimPatterns.yml @@ -1,2 +1,2 @@ slimPatterns: - - "*.egg-info*" \ No newline at end of file + - "**/*.egg-info*" diff --git a/tests/pipenv/_slimPatterns.yml b/tests/pipenv/_slimPatterns.yml index ffc3c134..70f863cc 100644 --- a/tests/pipenv/_slimPatterns.yml +++ b/tests/pipenv/_slimPatterns.yml @@ -1,2 +1,2 @@ slimPatterns: - - "*.egg-info*" \ No newline at end of file + - "**/*.egg-info*"