Skip to content

Commit cef29b5

Browse files
bsamuel-uidschep
bsamuel-ui
authored and
dschep
committed
Use JSZip 3.x to handle zip files (#178)
* Update to JSZip 3.x and remove zip-local - Restores ability to store unix permissions. - So async, much promises. * Fix some linter issues after merge. * Actually run prettier…
1 parent b695318 commit cef29b5

File tree

5 files changed

+222
-152
lines changed

5 files changed

+222
-152
lines changed

‎lib/inject.js

+76-88
Original file line numberDiff line numberDiff line change
@@ -1,97 +1,76 @@
1+
const BbPromise = require('bluebird');
12
const fse = require('fs-extra');
23
const glob = require('glob-all');
34
const get = require('lodash.get');
45
const set = require('lodash.set');
56
const path = require('path');
67
const values = require('lodash.values');
7-
const zipper = require('zip-local');
88
const JSZip = require('jszip');
9+
const { writeZip, zipFile } = require('./zipTree');
910

10-
/**
11-
* write zip contents to a file
12-
* @param {Object} zip
13-
* @param {string} path
14-
*/
15-
function writeZip(zip, path) {
16-
const buff = zip.generate({
17-
type: 'nodebuffer',
18-
compression: 'DEFLATE'
19-
});
20-
21-
fse.writeFileSync(path, buff);
22-
}
23-
24-
/**
25-
* add a new file to a zip file from a buffer
26-
* @param {Object} zip
27-
* @param {string} path path to put in zip
28-
* @param {string} buffer file contents
29-
*/
30-
function zipFile(zip, path, buffer) {
31-
zip.file(path, buffer, {
32-
date: new Date(0) // necessary to get the same hash when zipping the same content
33-
});
34-
}
11+
BbPromise.promisifyAll(fse);
3512

3613
/**
37-
* inject requirements into packaged application
14+
* Inject requirements into packaged application.
3815
* @param {string} requirementsPath requirements folder path
3916
* @param {string} packagePath target package path
4017
* @param {Object} options our options object
18+
* @return {Promise} the JSZip object constructed.
4119
*/
4220
function injectRequirements(requirementsPath, packagePath, options) {
4321
const noDeploy = new Set(options.noDeploy || []);
4422

45-
const zip = zipper.sync.unzip(packagePath).lowLevel();
46-
47-
glob
48-
.sync([path.join(requirementsPath, '**')], { mark: true, dot: true })
49-
.forEach(file => {
50-
if (file.endsWith('/')) {
51-
return;
52-
}
53-
54-
const relativeFile = path.relative(requirementsPath, file);
55-
56-
if (relativeFile.match(/^__pycache__[\\/]/)) {
57-
return;
58-
}
59-
if (noDeploy.has(relativeFile.split(/([-\\/]|\.py$|\.pyc$)/, 1)[0])) {
60-
return;
61-
}
62-
63-
zipFile(zip, relativeFile, fse.readFileSync(file));
64-
});
65-
66-
writeZip(zip, packagePath);
23+
return fse
24+
.readFileAsync(packagePath)
25+
.then(buffer => JSZip.loadAsync(buffer))
26+
.then(zip =>
27+
BbPromise.resolve(
28+
glob.sync([path.join(requirementsPath, '**')], {
29+
mark: true,
30+
dot: true
31+
})
32+
)
33+
.map(file => [file, path.relative(requirementsPath, file)])
34+
.filter(
35+
([file, relativeFile]) =>
36+
!file.endsWith('/') &&
37+
!relativeFile.match(/^__pycache__[\\/]/) &&
38+
!noDeploy.has(relativeFile.split(/([-\\/]|\.py$|\.pyc$)/, 1)[0])
39+
)
40+
.map(([file, relativeFile]) =>
41+
zipFile(zip, relativeFile, fse.readFileAsync(file))
42+
)
43+
.then(() => writeZip(zip, packagePath))
44+
);
6745
}
6846

6947
/**
70-
* remove all modules but the selected module from a package
71-
* @param {string} source original package
72-
* @param {string} target result package
48+
* Remove all modules but the selected module from a package.
49+
* @param {string} source path to original package
50+
* @param {string} target path to result package
7351
* @param {string} module module to keep
52+
* @return {Promise} the JSZip object written out.
7453
*/
7554
function moveModuleUp(source, target, module) {
76-
const sourceZip = zipper.sync.unzip(source).memory();
77-
const targetZip = JSZip.make();
78-
79-
sourceZip.contents().forEach(file => {
80-
if (!file.startsWith(module + '/')) {
81-
return;
82-
}
83-
zipFile(
84-
targetZip,
85-
file.replace(module + '/', ''),
86-
sourceZip.read(file, 'buffer')
87-
);
88-
});
89-
90-
writeZip(targetZip, target);
55+
const targetZip = new JSZip();
56+
57+
return fse
58+
.readFileAsync(source)
59+
.then(buffer => JSZip.loadAsync(buffer))
60+
.then(sourceZip => sourceZip.filter(file => file.startsWith(module + '/')))
61+
.map(srcZipObj =>
62+
zipFile(
63+
targetZip,
64+
srcZipObj.name.replace(module + '/', ''),
65+
srcZipObj.async('nodebuffer')
66+
)
67+
)
68+
.then(() => writeZip(targetZip, target));
9169
}
9270

9371
/**
94-
* inject requirements into packaged application
72+
* Inject requirements into packaged application.
73+
* @return {Promise} the combined promise for requirements injection.
9574
*/
9675
function injectAllRequirements() {
9776
this.serverless.cli.log('Injecting required Python packages to package...');
@@ -101,30 +80,39 @@ function injectAllRequirements() {
10180
}
10281

10382
if (this.serverless.service.package.individually) {
104-
values(this.serverless.service.functions).forEach(f => {
105-
if (
106-
!(f.runtime || this.serverless.service.provider.runtime).match(
83+
return BbPromise.resolve(values(this.serverless.service.functions))
84+
.filter(func =>
85+
(func.runtime || this.serverless.service.provider.runtime).match(
10786
/^python.*/
10887
)
109-
) {
110-
return;
111-
}
112-
if (!get(f, 'module')) {
113-
set(f, ['module'], '.');
114-
}
115-
if (f.module !== '.') {
116-
const artifactPath = path.join('.serverless', `${f.module}.zip`);
117-
moveModuleUp(f.package.artifact, artifactPath, f.module);
118-
f.package.artifact = artifactPath;
119-
}
120-
injectRequirements(
121-
path.join('.serverless', f.module, 'requirements'),
122-
f.package.artifact,
123-
this.options
88+
)
89+
.map(func => {
90+
if (!get(func, 'module')) {
91+
set(func, ['module'], '.');
92+
}
93+
return func;
94+
})
95+
.map(func => {
96+
if (func.module !== '.') {
97+
const artifact = func.package.artifact;
98+
const newArtifact = path.join('.serverless', `${func.module}.zip`);
99+
func.package.artifact = newArtifact;
100+
return moveModuleUp(artifact, newArtifact, func.module).then(
101+
() => func
102+
);
103+
} else {
104+
return func;
105+
}
106+
})
107+
.map(func =>
108+
injectRequirements(
109+
path.join('.serverless', func.module, 'requirements'),
110+
func.package.artifact,
111+
this.options
112+
)
124113
);
125-
});
126114
} else {
127-
injectRequirements(
115+
return injectRequirements(
128116
path.join('.serverless', 'requirements'),
129117
this.serverless.service.package.artifact,
130118
this.options

‎lib/zip.js

+55-61
Original file line numberDiff line numberDiff line change
@@ -2,47 +2,43 @@ const fse = require('fs-extra');
22
const path = require('path');
33
const get = require('lodash.get');
44
const set = require('lodash.set');
5-
const zipper = require('zip-local');
6-
const BbPromise = require('bluebird');
75
const values = require('lodash.values');
6+
const uniqBy = require('lodash.uniqby');
7+
const BbPromise = require('bluebird');
8+
const JSZip = require('jszip');
9+
const { addTree, writeZip } = require('./zipTree');
810

911
BbPromise.promisifyAll(fse);
1012

1113
/**
12-
* add the vendor helper to the current service tree
14+
* Add the vendor helper to the current service tree.
1315
* @return {Promise}
1416
*/
1517
function addVendorHelper() {
1618
if (this.options.zip) {
1719
if (this.serverless.service.package.individually) {
18-
let promises = [];
19-
let doneModules = [];
20-
values(this.serverless.service.functions).forEach(f => {
21-
if (!get(f, 'package.include')) {
22-
set(f, ['package', 'include'], []);
23-
}
24-
if (!get(f, 'module')) {
25-
set(f, ['module'], '.');
26-
}
27-
28-
f.package.include.push('unzip_requirements.py');
29-
30-
if (!doneModules.includes(f.module)) {
20+
return BbPromise.resolve(values(this.serverless.service.functions))
21+
.map(f => {
22+
if (!get(f, 'package.include')) {
23+
set(f, ['package', 'include'], []);
24+
}
25+
if (!get(f, 'module')) {
26+
set(f, ['module'], '.');
27+
}
28+
f.package.include.push('unzip_requirements.py');
29+
return f;
30+
})
31+
.then(functions => uniqBy(functions, func => func.module))
32+
.map(f => {
3133
this.serverless.cli.log(
3234
`Adding Python requirements helper to ${f.module}...`
3335
);
3436

35-
promises.push(
36-
fse.copyAsync(
37-
path.resolve(__dirname, '../unzip_requirements.py'),
38-
path.join(this.servicePath, f.module, 'unzip_requirements.py')
39-
)
37+
return fse.copyAsync(
38+
path.resolve(__dirname, '../unzip_requirements.py'),
39+
path.join(this.servicePath, f.module, 'unzip_requirements.py')
4040
);
41-
42-
doneModules.push(f.module);
43-
}
44-
});
45-
return BbPromise.all(promises);
41+
});
4642
} else {
4743
this.serverless.cli.log('Adding Python requirements helper...');
4844

@@ -61,31 +57,28 @@ function addVendorHelper() {
6157
}
6258

6359
/**
64-
* remove the vendor helper from the current service tree
65-
* @return {Promise}
60+
* Remove the vendor helper from the current service tree.
61+
* @return {Promise} the promise to remove the vendor helper.
6662
*/
6763
function removeVendorHelper() {
6864
if (this.options.zip && this.options.cleanupZipHelper) {
6965
if (this.serverless.service.package.individually) {
70-
let promises = [];
71-
let doneModules = [];
72-
values(this.serverless.service.functions).forEach(f => {
73-
if (!get(f, 'module')) {
74-
set(f, ['module'], '.');
75-
}
76-
if (!doneModules.includes(f.module)) {
66+
return BbPromise.resolve(values(this.serverless.service.functions))
67+
.map(f => {
68+
if (!get(f, 'module')) {
69+
set(f, ['module'], '.');
70+
}
71+
return f;
72+
})
73+
.then(funcs => uniqBy(funcs, f => f.module))
74+
.map(f => {
7775
this.serverless.cli.log(
7876
`Removing Python requirements helper from ${f.module}...`
7977
);
80-
promises.push(
81-
fse.removeAsync(
82-
path.join(this.servicePath, f.module, 'unzip_requirements.py')
83-
)
78+
return fse.removeAsync(
79+
path.join(this.servicePath, f.module, 'unzip_requirements.py')
8480
);
85-
doneModules.push(f.module);
86-
}
87-
});
88-
return BbPromise.all(promises);
81+
});
8982
} else {
9083
this.serverless.cli.log('Removing Python requirements helper...');
9184
return fse.removeAsync(
@@ -96,35 +89,36 @@ function removeVendorHelper() {
9689
}
9790

9891
/**
99-
* zip up .serverless/requirements
92+
* Zip up .serverless/requirements or .serverless/[MODULE]/requirements.
93+
* @return {Promise} the promise to pack requirements.
10094
*/
10195
function packRequirements() {
10296
if (this.options.zip) {
10397
if (this.serverless.service.package.individually) {
104-
let doneModules = [];
105-
values(this.serverless.service.functions).forEach(f => {
106-
if (!get(f, 'module')) {
107-
set(f, ['module'], '.');
108-
}
109-
if (!doneModules.includes(f.module)) {
98+
return BbPromise.resolve(values(this.serverless.service.functions))
99+
.map(f => {
100+
if (!get(f, 'module')) {
101+
set(f, ['module'], '.');
102+
}
103+
return f;
104+
})
105+
.then(funcs => uniqBy(funcs, f => f.module))
106+
.map(f => {
110107
this.serverless.cli.log(
111108
`Zipping required Python packages for ${f.module}...`
112109
);
113110
f.package.include.push(`${f.module}/.requirements.zip`);
114-
zipper.sync
115-
.zip(`.serverless/${f.module}/requirements`)
116-
.compress()
117-
.save(`${f.module}/.requirements.zip`);
118-
doneModules.push(f.module);
119-
}
120-
});
111+
return addTree(
112+
new JSZip(),
113+
`.serverless/${f.module}/requirements`
114+
).then(zip => writeZip(zip, `${f.module}/.requirements.zip`));
115+
});
121116
} else {
122117
this.serverless.cli.log('Zipping required Python packages...');
123118
this.serverless.service.package.include.push('.requirements.zip');
124-
zipper.sync
125-
.zip(path.join(this.servicePath, '.serverless/requirements'))
126-
.compress()
127-
.save(path.join(this.servicePath, '.requirements.zip'));
119+
return addTree(new JSZip(), '.serverless/requirements').then(zip =>
120+
writeZip(zip, path.join(this.servicePath, '.requirements.zip'))
121+
);
128122
}
129123
}
130124
}

0 commit comments

Comments
 (0)