Skip to content

Commit 190df96

Browse files
dschepdschep
authored
Merge pull request #303 from bweigel/209_caching_bug
Fixes weird bug, where root & submodule requirements weren't packaged correctly
2 parents 6255b16 + 5016ca8 commit 190df96

File tree

8 files changed

+198
-32
lines changed

8 files changed

+198
-32
lines changed

‎lib/pip.js

+19-25
Original file line numberDiff line numberDiff line change
@@ -53,15 +53,15 @@ function mergeCommands(commands) {
5353
* @param {Object} options
5454
* @return {undefined}
5555
*/
56-
function installRequirementsFile(
56+
function generateRequirementsFile(
5757
requirementsPath,
5858
targetFile,
5959
serverless,
6060
servicePath,
6161
options
6262
) {
6363
if (options.usePipenv && fse.existsSync(path.join(servicePath, 'Pipfile'))) {
64-
generateRequirementsFile(
64+
filterRequirementsFile(
6565
path.join(servicePath, '.serverless/requirements.txt'),
6666
targetFile,
6767
options
@@ -70,7 +70,7 @@ function installRequirementsFile(
7070
`Parsed requirements.txt from Pipfile in ${targetFile}...`
7171
);
7272
} else {
73-
generateRequirementsFile(requirementsPath, targetFile, options);
73+
filterRequirementsFile(requirementsPath, targetFile, options);
7474
serverless.cli.log(
7575
`Generated requirements from ${requirementsPath} in ${targetFile}...`
7676
);
@@ -306,7 +306,6 @@ function dockerPathForWin(path) {
306306
return path;
307307
}
308308
}
309-
310309
/** create a filtered requirements.txt without anything from noDeploy
311310
* then remove all comments and empty lines, and sort the list which
312311
* assist with matching the static cache. The sorting will skip any
@@ -318,7 +317,7 @@ function dockerPathForWin(path) {
318317
* @param {string} target requirements where results are written
319318
* @param {Object} options
320319
*/
321-
function generateRequirementsFile(source, target, options) {
320+
function filterRequirementsFile(source, target, options) {
322321
const noDeploy = new Set(options.noDeploy || []);
323322
const requirements = fse
324323
.readFileSync(source, { encoding: 'utf-8' })
@@ -413,11 +412,21 @@ function installRequirementsIfNeeded(
413412
}
414413
}
415414

416-
// First, generate the requirements file to our local .serverless folder
417-
fse.ensureDirSync(path.join(servicePath, '.serverless'));
418-
const slsReqsTxt = path.join(servicePath, '.serverless', 'requirements.txt');
415+
let requirementsTxtDirectory;
416+
// Copy our requirements to another path in .serverless (incase of individually packaged)
417+
if (modulePath && modulePath !== '.') {
418+
requirementsTxtDirectory = path.join(
419+
servicePath,
420+
'.serverless',
421+
modulePath
422+
);
423+
} else {
424+
requirementsTxtDirectory = path.join(servicePath, '.serverless');
425+
}
426+
fse.ensureDirSync(requirementsTxtDirectory);
427+
const slsReqsTxt = path.join(requirementsTxtDirectory, 'requirements.txt');
419428

420-
installRequirementsFile(
429+
generateRequirementsFile(
421430
fileName,
422431
slsReqsTxt,
423432
serverless,
@@ -433,28 +442,13 @@ function installRequirementsIfNeeded(
433442
return false;
434443
}
435444

436-
// Copy our requirements to another filename in .serverless (incase of individually packaged)
437-
if (modulePath && modulePath != '.') {
438-
fse.existsSync(path.join(servicePath, '.serverless', modulePath));
439-
const destinationFile = path.join(
440-
servicePath,
441-
'.serverless',
442-
modulePath,
443-
'requirements.txt'
444-
);
445-
serverless.cli.log(
446-
`Copying from ${slsReqsTxt} into ${destinationFile} ...`
447-
);
448-
fse.copySync(slsReqsTxt, destinationFile);
449-
}
450-
451445
// Then generate our MD5 Sum of this requirements file to determine where it should "go" to and/or pull cache from
452446
const reqChecksum = md5Path(slsReqsTxt);
453447

454448
// Then figure out where this cache should be, if we're caching, if we're in a module, etc
455449
const workingReqsFolder = getRequirementsWorkingPath(
456450
reqChecksum,
457-
servicePath,
451+
requirementsTxtDirectory,
458452
options
459453
);
460454

‎lib/shared.js

+6-2
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,11 @@ function checkForAndDeleteMaxCacheVersions(options, serverless) {
5757
* @param {Object} options
5858
* @return {string}
5959
*/
60-
function getRequirementsWorkingPath(subfolder, servicePath, options) {
60+
function getRequirementsWorkingPath(
61+
subfolder,
62+
requirementsTxtDirectory,
63+
options
64+
) {
6165
// If we want to use the static cache
6266
if (options && options.useStaticCache) {
6367
if (subfolder) {
@@ -69,7 +73,7 @@ function getRequirementsWorkingPath(subfolder, servicePath, options) {
6973
}
7074

7175
// If we don't want to use the static cache, then fallback to the way things used to work
72-
return path.join(servicePath, '.serverless', 'requirements');
76+
return path.join(requirementsTxtDirectory, 'requirements');
7377
}
7478

7579
/**

‎test.js

+157-1
Original file line numberDiff line numberDiff line change
@@ -1008,35 +1008,79 @@ test('py3.6 can package flask with package individually option', t => {
10081008
sls(['--individually=true', 'package']);
10091009

10101010
const zipfiles_hello = listZipFiles('.serverless/hello.zip');
1011+
t.false(
1012+
zipfiles_hello.includes(`fn2${sep}__init__.py`),
1013+
'fn2 is NOT packaged in function hello'
1014+
);
10111015
t.true(
10121016
zipfiles_hello.includes('handler.py'),
10131017
'handler.py is packaged in function hello'
10141018
);
1019+
t.false(
1020+
zipfiles_hello.includes(`dataclasses.py`),
1021+
'dataclasses is NOT packaged in function hello'
1022+
);
10151023
t.true(
10161024
zipfiles_hello.includes(`flask${sep}__init__.py`),
10171025
'flask is packaged in function hello'
10181026
);
10191027

10201028
const zipfiles_hello2 = listZipFiles('.serverless/hello2.zip');
1029+
t.false(
1030+
zipfiles_hello2.includes(`fn2${sep}__init__.py`),
1031+
'fn2 is NOT packaged in function hello2'
1032+
);
10211033
t.true(
10221034
zipfiles_hello2.includes('handler.py'),
10231035
'handler.py is packaged in function hello2'
10241036
);
1037+
t.false(
1038+
zipfiles_hello2.includes(`dataclasses.py`),
1039+
'dataclasses is NOT packaged in function hello2'
1040+
);
10251041
t.true(
10261042
zipfiles_hello2.includes(`flask${sep}__init__.py`),
10271043
'flask is packaged in function hello2'
10281044
);
10291045

10301046
const zipfiles_hello3 = listZipFiles('.serverless/hello3.zip');
1047+
t.false(
1048+
zipfiles_hello3.includes(`fn2${sep}__init__.py`),
1049+
'fn2 is NOT packaged in function hello3'
1050+
);
10311051
t.true(
10321052
zipfiles_hello3.includes('handler.py'),
1033-
'handler.py is packaged in function hello2'
1053+
'handler.py is packaged in function hello3'
1054+
);
1055+
t.false(
1056+
zipfiles_hello3.includes(`dataclasses.py`),
1057+
'dataclasses is NOT packaged in function hello3'
10341058
);
10351059
t.false(
10361060
zipfiles_hello3.includes(`flask${sep}__init__.py`),
10371061
'flask is NOT packaged in function hello3'
10381062
);
10391063

1064+
const zipfiles_hello4 = listZipFiles(
1065+
'.serverless/fn2-sls-py-req-test-dev-hello4.zip'
1066+
);
1067+
t.false(
1068+
zipfiles_hello4.includes(`fn2${sep}__init__.py`),
1069+
'fn2 is NOT packaged in function hello4'
1070+
);
1071+
t.true(
1072+
zipfiles_hello4.includes('fn2_handler.py'),
1073+
'fn2_handler is packaged in the zip-root in function hello4'
1074+
);
1075+
t.true(
1076+
zipfiles_hello4.includes(`dataclasses.py`),
1077+
'dataclasses is packaged in function hello4'
1078+
);
1079+
t.false(
1080+
zipfiles_hello4.includes(`flask${sep}__init__.py`),
1081+
'flask is NOT packaged in function hello4'
1082+
);
1083+
10401084
t.end();
10411085
});
10421086

@@ -1060,6 +1104,10 @@ test('py3.6 can package flask with package individually & slim option', t => {
10601104
zipfiles_hello.includes(`flask${sep}__init__.py`),
10611105
'flask is packaged in function hello'
10621106
);
1107+
t.false(
1108+
zipfiles_hello.includes(`dataclasses.py`),
1109+
'dataclasses is NOT packaged in function hello'
1110+
);
10631111

10641112
const zipfiles_hello2 = listZipFiles('.serverless/hello2.zip');
10651113
t.true(
@@ -1075,6 +1123,10 @@ test('py3.6 can package flask with package individually & slim option', t => {
10751123
zipfiles_hello2.includes(`flask${sep}__init__.py`),
10761124
'flask is packaged in function hello2'
10771125
);
1126+
t.false(
1127+
zipfiles_hello2.includes(`dataclasses.py`),
1128+
'dataclasses is NOT packaged in function hello2'
1129+
);
10781130

10791131
const zipfiles_hello3 = listZipFiles('.serverless/hello3.zip');
10801132
t.true(
@@ -1091,6 +1143,27 @@ test('py3.6 can package flask with package individually & slim option', t => {
10911143
'flask is NOT packaged in function hello3'
10921144
);
10931145

1146+
const zipfiles_hello4 = listZipFiles(
1147+
'.serverless/fn2-sls-py-req-test-dev-hello4.zip'
1148+
);
1149+
t.true(
1150+
zipfiles_hello4.includes('fn2_handler.py'),
1151+
'fn2_handler is packaged in the zip-root in function hello4'
1152+
);
1153+
t.true(
1154+
zipfiles_hello4.includes(`dataclasses.py`),
1155+
'dataclasses is packaged in function hello4'
1156+
);
1157+
t.false(
1158+
zipfiles_hello4.includes(`flask${sep}__init__.py`),
1159+
'flask is NOT packaged in function hello4'
1160+
);
1161+
t.deepEqual(
1162+
zipfiles_hello4.filter(filename => filename.endsWith('.pyc')),
1163+
[],
1164+
'no pyc files packaged in function hello4'
1165+
);
1166+
10941167
t.end();
10951168
});
10961169

@@ -1109,6 +1182,10 @@ test('py2.7 can package flask with package individually option', t => {
11091182
zipfiles_hello.includes(`flask${sep}__init__.py`),
11101183
'flask is packaged in function hello'
11111184
);
1185+
t.false(
1186+
zipfiles_hello.includes(`dataclasses.py`),
1187+
'dataclasses is NOT packaged in function hello'
1188+
);
11121189

11131190
const zipfiles_hello2 = listZipFiles('.serverless/hello2.zip');
11141191
t.true(
@@ -1119,6 +1196,10 @@ test('py2.7 can package flask with package individually option', t => {
11191196
zipfiles_hello2.includes(`flask${sep}__init__.py`),
11201197
'flask is packaged in function hello2'
11211198
);
1199+
t.false(
1200+
zipfiles_hello2.includes(`dataclasses.py`),
1201+
'dataclasses is NOT packaged in function hello2'
1202+
);
11221203

11231204
const zipfiles_hello3 = listZipFiles('.serverless/hello3.zip');
11241205
t.true(
@@ -1129,6 +1210,26 @@ test('py2.7 can package flask with package individually option', t => {
11291210
zipfiles_hello3.includes(`flask${sep}__init__.py`),
11301211
'flask is NOT packaged in function hello3'
11311212
);
1213+
t.false(
1214+
zipfiles_hello3.includes(`dataclasses.py`),
1215+
'dataclasses is NOT packaged in function hello3'
1216+
);
1217+
1218+
const zipfiles_hello4 = listZipFiles(
1219+
'.serverless/fn2-sls-py-req-test-dev-hello4.zip'
1220+
);
1221+
t.true(
1222+
zipfiles_hello4.includes('fn2_handler.py'),
1223+
'fn2_handler is packaged in the zip-root in function hello4'
1224+
);
1225+
t.true(
1226+
zipfiles_hello4.includes(`dataclasses.py`),
1227+
'dataclasses is packaged in function hello4'
1228+
);
1229+
t.false(
1230+
zipfiles_hello4.includes(`flask${sep}__init__.py`),
1231+
'flask is NOT packaged in function hello4'
1232+
);
11321233

11331234
t.end();
11341235
});
@@ -1153,6 +1254,10 @@ test('py2.7 can package flask with package individually & slim option', t => {
11531254
zipfiles_hello.includes(`flask${sep}__init__.py`),
11541255
'flask is packaged in function hello'
11551256
);
1257+
t.false(
1258+
zipfiles_hello.includes(`dataclasses.py`),
1259+
'dataclasses is NOT packaged in function hello'
1260+
);
11561261

11571262
const zipfiles_hello2 = listZipFiles('.serverless/hello2.zip');
11581263
t.true(
@@ -1168,6 +1273,10 @@ test('py2.7 can package flask with package individually & slim option', t => {
11681273
zipfiles_hello2.includes(`flask${sep}__init__.py`),
11691274
'flask is packaged in function hello2'
11701275
);
1276+
t.false(
1277+
zipfiles_hello2.includes(`dataclasses.py`),
1278+
'dataclasses is NOT packaged in function hello2'
1279+
);
11711280

11721281
const zipfiles_hello3 = listZipFiles('.serverless/hello3.zip');
11731282
t.true(
@@ -1183,6 +1292,26 @@ test('py2.7 can package flask with package individually & slim option', t => {
11831292
zipfiles_hello3.includes(`flask${sep}__init__.py`),
11841293
'flask is NOT packaged in function hello3'
11851294
);
1295+
t.false(
1296+
zipfiles_hello3.includes(`dataclasses.py`),
1297+
'dataclasses is NOT packaged in function hello3'
1298+
);
1299+
1300+
const zipfiles_hello4 = listZipFiles(
1301+
'.serverless/fn2-sls-py-req-test-dev-hello4.zip'
1302+
);
1303+
t.true(
1304+
zipfiles_hello4.includes('fn2_handler.py'),
1305+
'fn2_handler is packaged in the zip-root in function hello4'
1306+
);
1307+
t.true(
1308+
zipfiles_hello4.includes(`dataclasses.py`),
1309+
'dataclasses is packaged in function hello4'
1310+
);
1311+
t.false(
1312+
zipfiles_hello4.includes(`flask${sep}__init__.py`),
1313+
'flask is NOT packaged in function hello4'
1314+
);
11861315

11871316
t.end();
11881317
});
@@ -1255,6 +1384,10 @@ test('py3.6 can package lambda-decorators using vendor and invidiually option',
12551384
zipfiles_hello.includes(`lambda_decorators.py`),
12561385
'lambda_decorators.py is packaged in function hello'
12571386
);
1387+
t.false(
1388+
zipfiles_hello.includes(`dataclasses.py`),
1389+
'dataclasses is NOT packaged in function hello'
1390+
);
12581391

12591392
const zipfiles_hello2 = listZipFiles('.serverless/hello2.zip');
12601393
t.true(
@@ -1269,6 +1402,10 @@ test('py3.6 can package lambda-decorators using vendor and invidiually option',
12691402
zipfiles_hello2.includes(`lambda_decorators.py`),
12701403
'lambda_decorators.py is packaged in function hello2'
12711404
);
1405+
t.false(
1406+
zipfiles_hello2.includes(`dataclasses.py`),
1407+
'dataclasses is NOT packaged in function hello2'
1408+
);
12721409

12731410
const zipfiles_hello3 = listZipFiles('.serverless/hello3.zip');
12741411
t.true(
@@ -1283,7 +1420,26 @@ test('py3.6 can package lambda-decorators using vendor and invidiually option',
12831420
zipfiles_hello3.includes(`lambda_decorators.py`),
12841421
'lambda_decorators.py is NOT packaged in function hello3'
12851422
);
1423+
t.false(
1424+
zipfiles_hello3.includes(`dataclasses.py`),
1425+
'dataclasses is NOT packaged in function hello3'
1426+
);
12861427

1428+
const zipfiles_hello4 = listZipFiles(
1429+
'.serverless/fn2-sls-py-req-test-dev-hello4.zip'
1430+
);
1431+
t.true(
1432+
zipfiles_hello4.includes('fn2_handler.py'),
1433+
'fn2_handler is packaged in the zip-root in function hello4'
1434+
);
1435+
t.true(
1436+
zipfiles_hello4.includes(`dataclasses.py`),
1437+
'dataclasses is packaged in function hello4'
1438+
);
1439+
t.false(
1440+
zipfiles_hello4.includes(`flask${sep}__init__.py`),
1441+
'flask is NOT packaged in function hello4'
1442+
);
12871443
t.end();
12881444
});
12891445

‎tests/base/fn2/__init__.py

Whitespace-only changes.

‎tests/base/fn2/fn2_handler.py

Whitespace-only changes.

0 commit comments

Comments
 (0)