Skip to content

override slimPatterns instead of appending #276

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 19 commits into from
Nov 17, 2018
Merged
Show file tree
Hide file tree
Changes from 13 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 12 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -116,7 +116,7 @@ except ImportError:
_Works on non 'win32' environments: Docker, WSL are included_
To remove the tests, information and caches from the installed packages,
enable the `slim` option. This will: `strip` the `.so` files, remove `__pycache__`
directories and `dist-info` directories.
and `dist-info` directories as well as `.pyc` and `.pyo` files.
```yaml
custom:
pythonRequirements:
Expand All @@ -125,7 +125,8 @@ custom:
#### Custom Removal Patterns
To specify additional directories to remove from the installed packages,
define a list of patterns in the serverless config using the `slimPatterns`
option and glob syntax. Note, it matches against whole paths, so to match a file in any
option and glob syntax. These paterns will be added to the default ones (`**/*.py[c|o]`, `**/__pycache__*`, `**/*.dist-info*`).
Note, the glob syntax matches against whole paths, so to match a file in any
directory, start your pattern with `**/`.
```yaml
custom:
Expand All @@ -134,6 +135,15 @@ custom:
slimPatterns:
- "**/*.egg-info*"
```
To overwrite the default patterns set the option `slimPatternsAppendDefaults` to `false` (`true` by default).
```yaml
custom:
pythonRequirements:
slim: true
slimPatternsAppendDefaults: false
slimPatterns:
- "**/*.egg-info*"
```
This will remove all folders within the installed requirements that match
the names in `slimPatterns`
## Omitting Packages
Expand Down
1 change: 1 addition & 0 deletions index.js
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ class ServerlessPythonRequirements {
{
slim: false,
slimPatterns: false,
slimPatternsAppendDefaults: true,
zip: false,
cleanupZipHelper: true,
invalidateCaches: false,
Expand Down
9 changes: 8 additions & 1 deletion lib/slim.js
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,14 @@ const getStripCommand = (options, folderPath) =>
const deleteFiles = (options, folderPath) => {
let patterns = ['**/*.py[c|o]', '**/__pycache__*', '**/*.dist-info*'];
if (options.slimPatterns) {
patterns = patterns.concat(options.slimPatterns);
if (
options.slimPatternsAppendDefaults === false ||
options.slimPatternsAppendDefaults == 'false'
) {
patterns = options.slimPatterns;
} else {
patterns = patterns.concat(options.slimPatterns);
}
}
for (const pattern of patterns) {
for (const file of glob.sync(`${folderPath}/${pattern}`)) {
Expand Down
4 changes: 1 addition & 3 deletions test.bats
Original file line number Diff line number Diff line change
Expand Up @@ -297,7 +297,7 @@ teardown() {
cd tests/base
cat _slimPatterns.yml > slimPatterns.yml
npm i $(npm pack ../..)
sls --runtime=python2.7 --slim=true packag
sls --runtime=python2.7 --slim=true package
unzip .serverless/sls-py-req-test.zip -d puck
ls puck/flask
test $(find puck -name "*.pyc" | wc -l) -eq 0
Expand Down Expand Up @@ -462,7 +462,6 @@ teardown() {
test $(find "puck*" -name "*.pyc" | wc -l) -eq 0
}


@test "py2.7 can package flask with package individually option" {
cd tests/base
npm i $(npm pack ../..)
Expand All @@ -488,7 +487,6 @@ teardown() {
test $(find puck* -name "*.pyc" | wc -l) -eq 0
}


@test "py3.6 can package only requirements of module" {
cd tests/individually
npm i $(npm pack ../..)
Expand Down
155 changes: 148 additions & 7 deletions test.js
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ const deasync = require('deasync-promise');
const glob = require('glob-all');
const JSZip = require('jszip');
const tape = require('tape');
const { removeSync, readFileSync } = require('fs-extra');
const { removeSync, readFileSync, copySync } = require('fs-extra');
const { sep } = require('path');

const { getUserCachePath } = require('./lib/shared');
Expand Down Expand Up @@ -86,7 +86,7 @@ const listRequirementsZipFiles = filename => {
const zip = deasync(new JSZip().loadAsync(readFileSync(filename)));
const reqsBuffer = deasync(zip.file('.requirements.zip').async('nodebuffer'));
const reqsZip = deasync(new JSZip().loadAsync(reqsBuffer));
return Object.keys(reqsZip.files)
return Object.keys(reqsZip.files);
};

test('default pythonBin can package flask with default options', t => {
Expand Down Expand Up @@ -142,7 +142,6 @@ test('py3.6 can package flask with slim option', t => {
t.end();
});


/*
* News tests not in test.bats
*/
Expand All @@ -151,10 +150,18 @@ test("py3.6 doesn't package bottle with zip option", t => {
process.chdir('tests/base');
const path = npm(['pack', '../..']);
npm(['i', path]);
perl(['-p', "-i'.bak'", '-e', 's/(pythonRequirements:$)/\\1\\n noDeploy: [bottle]/', 'serverless.yml'])
perl([
'-p',
"-i'.bak'",
'-e',
's/(pythonRequirements:$)/\\1\\n noDeploy: [bottle]/',
'serverless.yml'
]);
sls([`--pythonBin=${getPythonBin(3)}`, '--zip=true', 'package']);
const zipfiles = listZipFiles('.serverless/sls-py-req-test.zip');
const zippedReqs = listRequirementsZipFiles('.serverless/sls-py-req-test.zip');
const zippedReqs = listRequirementsZipFiles(
'.serverless/sls-py-req-test.zip'
);
t.true(
zipfiles.includes('.requirements.zip'),
'zipped requirements are packaged'
Expand All @@ -164,7 +171,141 @@ test("py3.6 doesn't package bottle with zip option", t => {
zipfiles.includes(`flask${sep}__init__.py`),
"flask isn't packaged on its own"
);
t.true(zippedReqs.includes(`flask/__init__.py`), 'flask is packaged in the .requirements.zip file');
t.false(zippedReqs.includes(`bottle.py`), 'bottle is not packaged in the .requirements.zip file');
t.true(
zippedReqs.includes(`flask/__init__.py`),
'flask is packaged in the .requirements.zip file'
);
t.false(
zippedReqs.includes(`bottle.py`),
'bottle is not packaged in the .requirements.zip file'
);
t.end();
});

test('py3.6 can package flask with slim, slimPatterns & slimPatternsAppendDefaults=false options', t => {
process.chdir('tests/base');
copySync('_slimPatterns.yml', 'slimPatterns.yml');
const path = npm(['pack', '../..']);
npm(['i', path]);
sls(['--slim=true', '--slimPatternsAppendDefaults=false', 'package']);

const zipfiles = listZipFiles('.serverless/sls-py-req-test.zip');
t.true(zipfiles.includes(`flask${sep}__init__.py`), 'flask is packaged');
t.true(
zipfiles.filter(filename => filename.endsWith('.pyc')).length >= 1,
'pyc files are packaged'
);
t.deepEqual(
zipfiles.filter(filename => filename.includes('.egg-info')),
[],
'.egg-info folders are not packaged'
);
t.end();
});

test('py3.6 can package flask with slim & dockerizePip & slimPatterns & slimPatternsAppendDefaults=false options', t => {
process.chdir('tests/base');
copySync('_slimPatterns.yml', 'slimPatterns.yml');
const path = npm(['pack', '../..']);
npm(['i', path]);
/* TODO:
docker &> /dev/null || skip "docker not present"
! uname -sm|grep Linux || groups|grep docker || id -u|egrep '^0$' || skip "can't dockerize on linux if not root & not in docker group"
*/
sls([
'--dockerizePip=true',
'--slim=true',
'--slimPatternsAppendDefaults=false',
'package'
]);

const zipfiles = listZipFiles('.serverless/sls-py-req-test.zip');
t.true(zipfiles.includes(`flask${sep}__init__.py`), 'flask is packaged');
t.true(
zipfiles.filter(filename => filename.endsWith('.pyc')).length >= 1,
'pyc files are packaged'
);
t.deepEqual(
zipfiles.filter(filename => filename.includes('.egg-info')),
[],
'.egg-info folders are not packaged'
);
t.end();
});

test('py2.7 can package flask with slim & slimPatterns & slimPatternsAppendDefaults=false options', t => {
process.chdir('tests/base');
copySync('_slimPatterns.yml', 'slimPatterns.yml');
const path = npm(['pack', '../..']);
npm(['i', path]);
sls([
'--runtime=python2.7',
'--slim=true',
'--slimPatternsAppendDefaults=false',
'package'
]);

const zipfiles = listZipFiles('.serverless/sls-py-req-test.zip');
t.true(zipfiles.includes(`flask${sep}__init__.py`), 'flask is packaged');
t.true(
zipfiles.filter(filename => filename.endsWith('.pyc')).length >= 1,
'pyc files are packaged'
);
t.deepEqual(
zipfiles.filter(filename => filename.includes('.egg-info')),
[],
'.egg-info folders are not packaged'
);
t.end();
});

test('py2.7 can package flask with slim & dockerizePip & slimPatterns & slimPatternsAppendDefaults=false options', t => {
process.chdir('tests/base');
copySync('_slimPatterns.yml', 'slimPatterns.yml');
const path = npm(['pack', '../..']);
npm(['i', path]);
/* TODO:
docker &> /dev/null || skip "docker not present"
! uname -sm|grep Linux || groups|grep docker || id -u|egrep '^0$' || skip "can't dockerize on linux if not root & not in docker group"
*/
sls([
'--dockerizePip=true',
'--runtime=python2.7',
'--slim=true',
'--slimPatternsAppendDefaults=false',
'package'
]);
const zipfiles = listZipFiles('.serverless/sls-py-req-test.zip');
t.true(zipfiles.includes(`flask${sep}__init__.py`), 'flask is packaged');
t.true(
zipfiles.filter(filename => filename.endsWith('.pyc')).length >= 1,
'pyc files are packaged'
);
t.deepEqual(
zipfiles.filter(filename => filename.includes('.egg-info')),
[],
'.egg-info folders are not packaged'
);
t.end();
});

test('pipenv py3.6 can package flask with slim & slimPatterns & slimPatternsAppendDefaults=false option', t => {
process.chdir('tests/pipenv');
copySync('_slimPatterns.yml', 'slimPatterns.yml');
const path = npm(['pack', '../..']);
npm(['i', path]);

sls(['--slim=true', '--slimPatternsAppendDefaults=false', 'package']);
const zipfiles = listZipFiles('.serverless/sls-py-req-test.zip');
t.true(zipfiles.includes(`flask${sep}__init__.py`), 'flask is packaged');
t.true(
zipfiles.filter(filename => filename.endsWith('.pyc')).length >= 1,
'pyc files are packaged'
);
t.deepEqual(
zipfiles.filter(filename => filename.includes('.egg-info')),
[],
'.egg-info folders are not packaged'
);
t.end();
});
2 changes: 2 additions & 0 deletions tests/base/serverless.yml
Original file line number Diff line number Diff line change
Expand Up @@ -12,11 +12,13 @@ custom:
dockerizePip: ${opt:dockerizePip, self:custom.defaults.dockerizePip}
slim: ${opt:slim, self:custom.defaults.slim}
slimPatterns: ${file(./slimPatterns.yml):slimPatterns, self:custom.defaults.slimPatterns}
slimPatternsAppendDefaults: ${opt:slimPatternsAppendDefaults, self:custom.defaults.slimPatternsAppendDefaults}
vendor: ${opt:vendor, ''}
fileName: ${opt:fileName, 'requirements.txt'}
defaults:
slim: false
slimPatterns: false
slimPatternsAppendDefaults: true
zip: false
dockerizePip: false
individually: false
Expand Down
2 changes: 2 additions & 0 deletions tests/pipenv/serverless.yml
Original file line number Diff line number Diff line change
Expand Up @@ -11,10 +11,12 @@ custom:
zip: ${opt:zip, self:custom.defaults.zip}
slim: ${opt:slim, self:custom.defaults.slim}
slimPatterns: ${file(./slimPatterns.yml):slimPatterns, self:custom.defaults.slimPatterns}
slimPatternsAppendDefaults: ${opt:slimPatternsAppendDefaults, self:custom.defaults.slimPatternsAppendDefaults}
dockerizePip: ${opt:dockerizePip, self:custom.defaults.dockerizePip}
defaults:
zip: false
slimPatterns: false
slimPatternsAppendDefaults: true
slim: false
dockerizePip: false

Expand Down