Skip to content

Commit cc68a1f

Browse files
vladarwardpeetpieh
authored
feat(gatsby-plugin-sharp): add experimental opt-in lazy image processing mode for gatsby develop (#28288)
* feat(gatsby-plugin-sharp): enable lazy image processing * fix progress bar and cloud * track usage of query on demand feature * Use GATSBY_EXPERIMENTAL_LAZY_IMAGES vs `experimentalDisableLazyProcessing` plugin option * Don't hook to express when disabled * Update test snapshot * fix: restore behavior from master for onCreateDevServer * Add a note on FAST_DEV compatibility with gatsby-plugin-sharp * Reset GATSBY_EXPERIMENTAL_LAZY_IMAGES for non-develop and CI Co-authored-by: Ward Peeters <[email protected]> Co-authored-by: Michal Piechowiak <[email protected]>
1 parent 3666130 commit cc68a1f

File tree

6 files changed

+174
-5
lines changed

6 files changed

+174
-5
lines changed

‎packages/gatsby-plugin-sharp/package.json

+1
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
"bluebird": "^3.7.2",
1313
"fs-extra": "^9.0.1",
1414
"gatsby-core-utils": "^1.6.0-next.0",
15+
"gatsby-telemetry": "^1.6.0-next.1",
1516
"got": "^10.7.0",
1617
"imagemin": "^7.0.1",
1718
"imagemin-mozjpeg": "^9.0.0",

‎packages/gatsby-plugin-sharp/src/__tests__/__snapshots__/index.js.snap

+25
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,7 @@ exports[`gatsby-plugin-sharp fixed correctly infers the width when only the heig
5858
Array [
5959
Object {
6060
"args": Object {
61+
"isLazy": false,
6162
"operations": Array [
6263
Object {
6364
"args": Object {
@@ -114,6 +115,7 @@ exports[`gatsby-plugin-sharp fixed does not warn when the requested width is equ
114115
Array [
115116
Object {
116117
"args": Object {
118+
"isLazy": false,
117119
"operations": Array [
118120
Object {
119121
"args": Object {
@@ -170,6 +172,7 @@ exports[`gatsby-plugin-sharp fixed should give the same result with createJob as
170172
Array [
171173
Object {
172174
"args": Object {
175+
"isLazy": false,
173176
"operations": Array [
174177
Object {
175178
"args": Object {
@@ -226,6 +229,7 @@ exports[`gatsby-plugin-sharp fixed warns when the requested width is greater tha
226229
Array [
227230
Object {
228231
"args": Object {
232+
"isLazy": false,
229233
"operations": Array [
230234
Object {
231235
"args": Object {
@@ -268,6 +272,7 @@ exports[`gatsby-plugin-sharp fluid accepts srcSet breakpoints 1`] = `
268272
Array [
269273
Object {
270274
"args": Object {
275+
"isLazy": false,
271276
"operations": Array [
272277
Object {
273278
"args": Object {
@@ -338,6 +343,7 @@ exports[`gatsby-plugin-sharp fluid adds pathPrefix if defined 1`] = `
338343
Array [
339344
Object {
340345
"args": Object {
346+
"isLazy": false,
341347
"operations": Array [
342348
Object {
343349
"args": Object {
@@ -379,6 +385,7 @@ Array [
379385
Array [
380386
Object {
381387
"args": Object {
388+
"isLazy": false,
382389
"operations": Array [
383390
Object {
384391
"args": Object {
@@ -454,6 +461,7 @@ Array [
454461
Array [
455462
Object {
456463
"args": Object {
464+
"isLazy": false,
457465
"operations": Array [
458466
Object {
459467
"args": Object {
@@ -535,6 +543,7 @@ Array [
535543
Array [
536544
Object {
537545
"args": Object {
546+
"isLazy": false,
538547
"operations": Array [
539548
Object {
540549
"args": Object {
@@ -616,6 +625,7 @@ Array [
616625
Array [
617626
Object {
618627
"args": Object {
628+
"isLazy": false,
619629
"operations": Array [
620630
Object {
621631
"args": Object {
@@ -697,6 +707,7 @@ Array [
697707
Array [
698708
Object {
699709
"args": Object {
710+
"isLazy": false,
700711
"operations": Array [
701712
Object {
702713
"args": Object {
@@ -756,6 +767,7 @@ Array [
756767
Array [
757768
Object {
758769
"args": Object {
770+
"isLazy": false,
759771
"operations": Array [
760772
Object {
761773
"args": Object {
@@ -819,6 +831,7 @@ Array [
819831
Array [
820832
Object {
821833
"args": Object {
834+
"isLazy": false,
822835
"operations": Array [
823836
Object {
824837
"args": Object {
@@ -882,6 +895,7 @@ Array [
882895
Array [
883896
Object {
884897
"args": Object {
898+
"isLazy": false,
885899
"operations": Array [
886900
Object {
887901
"args": Object {
@@ -946,6 +960,7 @@ exports[`gatsby-plugin-sharp fluid does not change the arguments object it is gi
946960
Array [
947961
Object {
948962
"args": Object {
963+
"isLazy": false,
949964
"operations": Array [
950965
Object {
951966
"args": Object {
@@ -988,6 +1003,7 @@ exports[`gatsby-plugin-sharp fluid ensure maxWidth is in srcSet breakpoints 1`]
9881003
Array [
9891004
Object {
9901005
"args": Object {
1006+
"isLazy": false,
9911007
"operations": Array [
9921008
Object {
9931009
"args": Object {
@@ -1092,6 +1108,7 @@ exports[`gatsby-plugin-sharp fluid infers the maxWidth if only maxHeight is give
10921108
Array [
10931109
Object {
10941110
"args": Object {
1111+
"isLazy": false,
10951112
"operations": Array [
10961113
Object {
10971114
"args": Object {
@@ -1169,6 +1186,7 @@ exports[`gatsby-plugin-sharp fluid keeps original file name 1`] = `
11691186
Array [
11701187
Object {
11711188
"args": Object {
1189+
"isLazy": false,
11721190
"operations": Array [
11731191
Object {
11741192
"args": Object {
@@ -1211,6 +1229,7 @@ exports[`gatsby-plugin-sharp fluid prevents duplicate breakpoints 1`] = `
12111229
Array [
12121230
Object {
12131231
"args": Object {
1232+
"isLazy": false,
12141233
"operations": Array [
12151234
Object {
12161235
"args": Object {
@@ -1274,6 +1293,7 @@ exports[`gatsby-plugin-sharp fluid reject any breakpoints larger than the origin
12741293
Array [
12751294
Object {
12761295
"args": Object {
1296+
"isLazy": false,
12771297
"operations": Array [
12781298
Object {
12791299
"args": Object {
@@ -1344,6 +1364,7 @@ exports[`gatsby-plugin-sharp fluid should give the same result with createJob as
13441364
Array [
13451365
Object {
13461366
"args": Object {
1367+
"isLazy": false,
13471368
"operations": Array [
13481369
Object {
13491370
"args": Object {
@@ -1401,6 +1422,7 @@ exports[`gatsby-plugin-sharp queueImageResizing with createJob file name works w
14011422
Array [
14021423
Object {
14031424
"args": Object {
1425+
"isLazy": false,
14041426
"operations": Array [
14051427
Object {
14061428
"args": Object {
@@ -1445,6 +1467,7 @@ exports[`gatsby-plugin-sharp queueImageResizing with createJob should round heig
14451467
Array [
14461468
Object {
14471469
"args": Object {
1470+
"isLazy": false,
14481471
"operations": Array [
14491472
Object {
14501473
"args": Object {
@@ -1489,6 +1512,7 @@ exports[`gatsby-plugin-sharp queueImageResizing with createJobV2 file name works
14891512
Array [
14901513
Object {
14911514
"args": Object {
1515+
"isLazy": false,
14921516
"operations": Array [
14931517
Object {
14941518
"args": Object {
@@ -1531,6 +1555,7 @@ exports[`gatsby-plugin-sharp queueImageResizing with createJobV2 should round he
15311555
Array [
15321556
Object {
15331557
"args": Object {
1558+
"isLazy": false,
15341559
"operations": Array [
15351560
Object {
15361561
"args": Object {

‎packages/gatsby-plugin-sharp/src/gatsby-node.js

+123-2
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,45 @@ const {
22
setBoundActionCreators,
33
// queue: jobQueue,
44
// reportError,
5+
_unstable_createJob,
56
} = require(`./index`)
7+
const { pathExists } = require(`fs-extra`)
8+
const { slash, isCI } = require(`gatsby-core-utils`)
9+
const { trackFeatureIsUsed } = require(`gatsby-telemetry`)
610
const { getProgressBar, createOrGetProgressBar } = require(`./utils`)
711

812
const { setPluginOptions } = require(`./plugin-options`)
13+
const path = require(`path`)
14+
15+
function prepareLazyImagesExperiment(reporter) {
16+
if (!process.env.GATSBY_EXPERIMENTAL_LAZY_IMAGES) {
17+
return
18+
}
19+
if (process.env.gatsby_executing_command !== `develop`) {
20+
// We don't want to ever have this flag enabled for anything other than develop
21+
// in case someone have this env var globally set
22+
delete process.env.GATSBY_EXPERIMENTAL_LAZY_IMAGES
23+
return
24+
}
25+
if (isCI()) {
26+
delete process.env.GATSBY_EXPERIMENTAL_LAZY_IMAGES
27+
reporter.warn(
28+
`Lazy Image Processing experiment is not available in CI environment. Continuing with regular mode.`
29+
)
30+
return
31+
}
32+
// We show a different notice for GATSBY_EXPERIMENTAL_FAST_DEV umbrella
33+
if (!process.env.GATSBY_EXPERIMENTAL_FAST_DEV) {
34+
reporter.info(
35+
`[gatsby-plugin-sharp] The lazy image processing experiment is enabled`
36+
)
37+
}
38+
trackFeatureIsUsed(`LazyImageProcessing`)
39+
}
40+
41+
exports.onPreInit = ({ reporter }) => {
42+
prepareLazyImagesExperiment(reporter)
43+
}
944

1045
// create the progressbar once and it will be killed in another lifecycle
1146
const finishProgressBar = () => {
@@ -16,9 +51,62 @@ const finishProgressBar = () => {
1651
}
1752

1853
exports.onPostBuild = () => finishProgressBar()
19-
exports.onCreateDevServer = () => finishProgressBar()
2054

21-
exports.onPreBootstrap = ({ actions, emitter, reporter }, pluginOptions) => {
55+
exports.onCreateDevServer = async ({ app, cache, reporter }) => {
56+
if (!process.env.GATSBY_EXPERIMENTAL_LAZY_IMAGES) {
57+
finishProgressBar()
58+
return
59+
}
60+
61+
createOrGetProgressBar()
62+
finishProgressBar()
63+
64+
app.use(async (req, res, next) => {
65+
const pathOnDisk = path.resolve(path.join(`./public/`, req.url))
66+
67+
if (await pathExists(pathOnDisk)) {
68+
return res.sendFile(pathOnDisk)
69+
}
70+
71+
const jobContentDigest = await cache.get(req.url)
72+
const cacheResult = jobContentDigest
73+
? await cache.get(jobContentDigest)
74+
: null
75+
76+
if (!cacheResult) {
77+
return next()
78+
}
79+
80+
await _unstable_createJob(cacheResult, { reporter })
81+
// we should implement cache.del inside our abstraction
82+
await cache.cache.del(jobContentDigest)
83+
await cache.cache.del(req.url)
84+
85+
return res.sendFile(pathOnDisk)
86+
})
87+
}
88+
89+
// So something is wrong with the reporter, when I do this in preBootstrap,
90+
// the progressbar gets not updated
91+
exports.onPostBootstrap = async ({ reporter, cache, store }) => {
92+
if (process.env.gatsby_executing_command !== `develop`) {
93+
// recreate jobs that haven't been triggered by develop yet
94+
// removing stale jobs has already kicked in so we know these still need to process
95+
for (const [contentDigest] of store.getState().jobsV2.complete) {
96+
const job = await cache.get(contentDigest)
97+
98+
if (job) {
99+
// we dont have to await, gatsby does this for us
100+
_unstable_createJob(job, { reporter })
101+
}
102+
}
103+
}
104+
}
105+
106+
exports.onPreBootstrap = async (
107+
{ actions, emitter, reporter, cache, store },
108+
pluginOptions
109+
) => {
22110
setBoundActionCreators(actions)
23111
setPluginOptions(pluginOptions)
24112

@@ -40,6 +128,33 @@ exports.onPreBootstrap = ({ actions, emitter, reporter }, pluginOptions) => {
40128

41129
emitter.on(`CREATE_JOB_V2`, action => {
42130
if (action.plugin.name === `gatsby-plugin-sharp`) {
131+
if (action.payload.job.args.isLazy) {
132+
// we have to remove some internal pieces
133+
const job = {
134+
name: action.payload.job.name,
135+
inputPaths: action.payload.job.inputPaths.map(input => input.path),
136+
outputDir: action.payload.job.outputDir,
137+
args: {
138+
...action.payload.job.args,
139+
isLazy: false,
140+
},
141+
}
142+
cache.set(action.payload.job.contentDigest, job)
143+
144+
action.payload.job.args.operations.forEach(op => {
145+
const cacheKey = slash(
146+
path.relative(
147+
path.join(process.cwd(), `public`),
148+
path.join(action.payload.job.outputDir, op.outputPath)
149+
)
150+
)
151+
152+
cache.set(`/${cacheKey}`, action.payload.job.contentDigest)
153+
})
154+
155+
return
156+
}
157+
43158
const job = action.payload.job
44159
const imageCount = job.args.operations.length
45160
imageCountInJobsMap.set(job.contentDigest, imageCount)
@@ -51,6 +166,12 @@ exports.onPreBootstrap = ({ actions, emitter, reporter }, pluginOptions) => {
51166
emitter.on(`END_JOB_V2`, action => {
52167
if (action.plugin.name === `gatsby-plugin-sharp`) {
53168
const jobContentDigest = action.payload.jobContentDigest
169+
170+
// when it's lazy we didn't set it
171+
if (!imageCountInJobsMap.has(jobContentDigest)) {
172+
return
173+
}
174+
54175
const imageCount = imageCountInJobsMap.get(jobContentDigest)
55176
const progress = createOrGetProgressBar(reporter)
56177
progress.tick(imageCount)

‎packages/gatsby-plugin-sharp/src/gatsby-worker.js

+7-2
Original file line numberDiff line numberDiff line change
@@ -38,8 +38,12 @@ const q = queue(
3838
* @param {{inputPaths: string[], outputDir: string, args: WorkerInput}} args
3939
* @return Promise
4040
*/
41-
exports.IMAGE_PROCESSING = ({ inputPaths, outputDir, args }) =>
42-
new Promise((resolve, reject) => {
41+
exports.IMAGE_PROCESSING = ({ inputPaths, outputDir, args }) => {
42+
if (args.isLazy) {
43+
return Promise.resolve()
44+
}
45+
46+
return new Promise((resolve, reject) => {
4347
q.push({ inputPaths, outputDir, args }, function (err) {
4448
if (err) {
4549
return reject(err)
@@ -48,3 +52,4 @@ exports.IMAGE_PROCESSING = ({ inputPaths, outputDir, args }) =>
4852
return resolve()
4953
})
5054
})
55+
}

0 commit comments

Comments
 (0)